def provision_rabbitmq(admin_password):
append("/etc/apt/sources.list.d/rabbitmq.list",
"deb http://www.rabbitmq.com/debian/ testing main", use_sudo=True)
if not file_exists("/usr/sbin/rabbitmq-server"):
sudo("wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc")
sudo("apt-key add rabbitmq-signing-key-public.asc")
map(package_ensure_apt, ["rabbitmq-server", "rsync"])
dir_ensure("/etc/rabbitmq/rabbitmq.conf.d")
put("./conf/bunny/bunny.conf", "/etc/rabbitmq/rabbitmq.conf.d/", use_sudo=True)
sudo("chown -R rabbitmq.rabbitmq /srv/rabbitmq")
dir_ensure("/srv/rabbitmq/log", owner="rabbitmq", group="rabbitmq")
sudo("rm -rf /var/lib/rabbitmq")
sudo("rm -rf /var/log/rabbitmq")
with mode_sudo():
file_link("/srv/rabbitmq", "/var/lib/rabbitmq", owner="rabbitmq", group="rabbitmq")
file_link("/srv/rabbitmq/log", "/var/log/rabbitmq", owner="rabbitmq", group="rabbitmq")
sudo("service rabbitmq-server start")
sudo("rabbitmq-plugins enable rabbitmq_management")
sudo("rabbitmqctl add_user admin " + admin_password)
sudo("rabbitmqctl set_user_tags admin administrator")
sudo('rabbitmqctl set_permissions -p / admin ".*" ".*" ".*"')
sudo("rabbitmqctl delete_user guest")
sudo("service rabbitmq-server restart")
else:
print "skipped install, already have /usr/sbin/rabbitmq-server"
def setup_system():
user_ensure(project_name, home='/home/' + project_name)
if not dir_exists('/home/%s/production' % project_name):
with cd('/home/%s' % project_name):
run('git clone file:///root/repos/%s.git production' % project_name
)
run('git clone file:///root/repos/%s.git staging' % project_name)
with cd('production'):
run('virtualenv env')
run_venv('pip install -r assets/requirements.txt',
env='production')
with cd('%s/settings' % project_name):
file_link('production.py', '__init__.py')
run('mkdir -p logs')
run('chown %s:%s logs -R' % (project_name, project_name))
run('chmod o+wx logs -R')
run('mkdir -p static')
run_venv('./manage.py collectstatic --noinput',
env="production")
with cd('staging'):
run('virtualenv env')
run_venv('pip install -r assets/requirements.txt',
env='staging')
with cd('%s/settings' % project_name):
file_link('staging.py', '__init__.py')
run('mkdir -p logs')
run('chown %s:%s logs -R' % (project_name, project_name))
run('chmod o+wx logs -R')
run('mkdir -p static')
run_venv('./manage.py collectstatic --noinput',
env="staging")
def custom_packages():
notify('Installing custom packages.')
notify('Installing PhantomJS')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_USER_HOME'])
with cd(MACHINE['DIR_USER_HOME']):
sudo(
'wget -N https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2 -O phantomjs.tar.bz2'
)
sudo('tar jxvf phantomjs.tar.bz2')
sudo('mv phantomjs-1.9.1-linux-x86_64 /etc/phantomjs')
cuisine.file_link('/etc/phantomjs/bin/phantomjs',
'/usr/bin/phantomjs',
symbolic=True)
notify('Installing CasperJS')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_USER_HOME'])
with cd(MACHINE['DIR_USER_HOME']):
sudo(
'wget -N https://codeload.github.com/n1k0/casperjs/legacy.tar.gz/1.0.3 -O casperjs.tar.bz2'
)
sudo('tar xzvf casperjs.tar.bz2')
sudo('mv n1k0-casperjs-76fc831 /etc/casperjs')
cuisine.file_link('/etc/casperjs/bin/casperjs',
'/usr/bin/casperjs',
symbolic=True)
def link_supervisor():
# TODO: would be beter as sudo
env.user=ROOT_USER
env.password=ROOT_PASS
if not file_exists("/etc/supervisor/conf.d/rtf.conf"):
with cd("/etc/supervisor/conf.d"):
file_link("/opt/rtd/apps/readthedocs/current/readthedocs.org/conf/supervisor.conf", "rtf.conf")
def link_conf():
notify('Configuring necessary symlinks for our libraries.')
cuisine.mode_sudo()
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libjpeg.so', '/usr/lib/libjpeg.so', symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libpng.so', '/usr/lib/libpng.so', symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libz.so', '/usr/lib/libz.so', symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libfreetype.so', '/usr/lib/libfreetype.so', symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/liblcms.so', '/usr/lib/liblcms.so', symbolic=True)
def link_nginx():
# TODO: would be better as sudo
env.user=ROOT_USER
env.password=ROOT_PASS
if not file_exists("/etc/nginx/sites-enabled/rtf.conf"):
with cd("/etc/nginx/sites-enabled"):
file_link("/opt/rtd/apps/readthedocs/current/readthedocs.org/conf/nginx.conf", "rtf.conf")
def solr_ensure(project_path, venv_path='.venv'):
with mode_sudo():
package_ensure('openjdk-7-jdk libxml2-dev libxslt1-dev python-dev')
dir_ensure('/usr/java')
file_link('/usr/lib/jvm/java-7-openjdk-amd64', '/usr/java/default')
package_ensure('solr-tomcat')
with virtualenv(project_path, venv_path):
python_package_ensure('pysolr lxml cssselect')
def dir_symlink_created(src, dest, symbolic=True, mode=None, owner=None, group=None):
"""
Ensure that a symlink at `dest` pointing to `src` is created.
:param src: *required* str; path to the source real directory
:param dest: *required* str; path to the destination symlink
"""
cuisine.file_link(src, dest, symbolic=symbolic, mode=mode, owner=owner, group=group)
cuisine.file_is_link(src)
def link_supervisor():
# TODO: would be beter as sudo
env.user = ROOT_USER
env.password = ROOT_PASS
if not file_exists("/etc/supervisor/conf.d/rtf.conf"):
with cd("/etc/supervisor/conf.d"):
file_link(
"/opt/rtd/apps/readthedocs/current/readthedocs.org/conf/supervisor.conf",
"rtf.conf")
def link_nginx():
# TODO: would be better as sudo
env.user = ROOT_USER
env.password = ROOT_PASS
if not file_exists("/etc/nginx/sites-enabled/rtf.conf"):
with cd("/etc/nginx/sites-enabled"):
file_link(
"/opt/rtd/apps/readthedocs/current/readthedocs.org/conf/nginx.conf",
"rtf.conf")
def setup_apache():
# apache: vhost
file_upload('/etc/apache2/sites-available/%s' % project_name,
'assets/vhost.conf')
file_link('/etc/apache2/sites-available/%s' % project_name,
'/etc/apache2/sites-enabled/%s' % project_name)
# apache: enable mod_rewrite
run('a2enmod rewrite')
# apache: restart
run('service apache2 restart')
def create_symlink(self):
(basedir, symlink_location) = self.www_dir.rsplit('/', 1)
with mode_sudo():
dir_ensure(basedir,
recursive=True,
)
with cd(basedir):
with mode_sudo():
file_link(
os.sep.join((
self.repo_dir,
self.repo_django_root
)),
symlink_location)
def make_virtualenv():
""" builds project in virtual environment """
env.user=RTD_USER
env.password=RTD_PASS
# build the virtualenv
with cd("/opt/rtd/apps/readthedocs"):
if not dir_exists("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION):
run("virtualenv %s" % RTD_INITIAL_VERSION)
if not dir_exists("/opt/rtd/apps/readthedocs/current"):
file_link("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION, "/opt/rtd/apps/readthedocs/current")
# clone the repo
with cd("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION):
if not dir_exists("/opt/rtd/apps/readthedocs/%s/%s" % (RTD_INITIAL_VERSION, RTD_CLONE_NAME)):
run("git clone %s %s" % ( RTD_CLONE, RTD_CLONE_NAME) )
def link_conf():
notify('Configuring necessary symlinks for our libraries.')
cuisine.mode_sudo()
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libjpeg.so',
'/usr/lib/libjpeg.so',
symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libpng.so',
'/usr/lib/libpng.so',
symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libz.so',
'/usr/lib/libz.so',
symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/libfreetype.so',
'/usr/lib/libfreetype.so',
symbolic=True)
cuisine.file_link('/usr/lib/x86_64-linux-gnu/liblcms.so',
'/usr/lib/liblcms.so',
symbolic=True)
def make_virtualenv():
""" builds project in virtual environment """
env.user = RTD_USER
env.password = RTD_PASS
# build the virtualenv
with cd("/opt/rtd/apps/readthedocs"):
if not dir_exists(
"/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION):
run("virtualenv %s" % RTD_INITIAL_VERSION)
if not dir_exists("/opt/rtd/apps/readthedocs/current"):
file_link("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION,
"/opt/rtd/apps/readthedocs/current")
# clone the repo
with cd("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION):
if not dir_exists("/opt/rtd/apps/readthedocs/%s/%s" %
(RTD_INITIAL_VERSION, RTD_CLONE_NAME)):
run("git clone %s %s" % (RTD_CLONE, RTD_CLONE_NAME))
def provision():
"""Sets up packages and the deploy tree"""
mode_sudo()
for p in DEBIAN_PACKAGES:
package_ensure(p)
# Create the deploy tree
dir_ensure("/srv", owner="root", group="root")
ensure_tree('/srv', DEPLOY_TREE, owner=env.remote_user, group=env.remote_group)
# Postgres
dir_ensure("/etc/postgresql/8.4/main", owner="postgres", group="postgres")
provision_file_upload("/etc/postgresql/8.4/main/pg_hba.conf", mode='644')
provision_file_upload("/etc/postgresql/8.4/main/postgresql.conf", mode='644')
# nginx
provision_file_upload("/etc/nginx/nginx.conf", mode='644', owner='root', group='root')
provision_file_upload("/etc/nginx/sites-available/%(app)s" % env, mode='644', owner='root', group='root')
file_link("/etc/nginx/sites-available/%(app)s" % env, "/etc/nginx/sites-enabled/%(app)s" % env)
# set up the virtualenv
if not file_exists(env.virtualenv):
sudo('easy_install pip')
sudo('pip install virtualenv')
run('sudo -u %(remote_user)s virtualenv %(virtualenv)s' % env, pty=True)
# gunicorn
dir_ensure("/etc/gunicorn", owner="root", group="root")
provision_file_upload("/etc/gunicorn/%(app)s.conf.py" % env, mode='644', owner='root', group='root')
# supervisord
provision_file_upload("/etc/supervisor/conf.d/%(app)s.conf" % env, mode='644', owner='root', group='root')
# app init script
provision_file_upload("/etc/init.d/%(app)s" % env, mode='755', owner="root", group="root")
# poor man's start on boot
provision_file_upload("/etc/rc.local", mode='755', owner="root", group="root")
# settings_local.py
provision_file_upload("/srv/%(app)s/shared/settings_local.py", owner=env.remote_user, group=env.remote_group)
def apply(self, computer):
username = "******"
if not cuisine.user_check(username):
cuisine.user_create(username)
cuisine.sudo('usermod -p `openssl rand -base64 32` %s' % username)
key = self.environment["users"][username]["ssh_public_key"]
cuisine.ssh_authorize(username, key)
cuisine.dir_ensure('/home/example_com/www',
mode='755',
owner=username,
group=username)
cuisine.file_link('/home/example_com/www', '/srv/www/example_com')
self.push_package_file_set('example_com', computer)
cuisine.file_link('/etc/nginx/sites-available/example_com',
'/etc/nginx/sites-enabled/example_com')
cuisine.sudo("service nginx restart")
def custom_packages():
notify('Installing custom packages.')
notify('Installing PhantomJS')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_USER_HOME'])
with cd(MACHINE['DIR_USER_HOME']):
sudo('wget -N https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2 -O phantomjs.tar.bz2')
sudo('tar jxvf phantomjs.tar.bz2')
sudo('mv phantomjs-1.9.1-linux-x86_64 /etc/phantomjs')
cuisine.file_link('/etc/phantomjs/bin/phantomjs',
'/usr/bin/phantomjs', symbolic=True)
notify('Installing CasperJS')
cuisine.mode_sudo()
cuisine.dir_ensure(MACHINE['DIR_USER_HOME'])
with cd(MACHINE['DIR_USER_HOME']):
sudo('wget -N https://codeload.github.com/n1k0/casperjs/legacy.tar.gz/1.0.3 -O casperjs.tar.bz2')
sudo('tar xzvf casperjs.tar.bz2')
sudo('mv n1k0-casperjs-76fc831 /etc/casperjs')
cuisine.file_link('/etc/casperjs/bin/casperjs',
'/usr/bin/casperjs', symbolic=True)
def setup_vhosts():
simple_vhosts = (
("bengoa", "bengoa.co.uk", "bengoa.com"),
("cernpoi", "cerntourism.dvdbng.com"),
("4sq", "4sq.dvdbng.com"),
("eureka", "redeureka.es"),
("hoyga", "hoyga.com"),
("brweb", "bengoarocandio.com", "dvdbng.com"),
("brweb/david", "david.bengoarocandio.com"),
)
for vhost in simple_vhosts:
name, domains = vhost[0], vhost[1:]
main_domain = domains[0]
alias = list(domains[1:])
for domain in domains:
alias.append("www.%s" % domain)
upload_template("../templates/simple_vhost", "/etc/apache2/sites-available/%s" % main_domain, {
"NAME": name,
"DOMAIN": main_domain,
"ALIAS": " ".join(alias)
}, use_sudo=True)
if not cuisine.file_exists("/etc/apache2/sites-enabled/%s" % main_domain):
cuisine.file_link("/etc/apache2/sites-available/%s" % main_domain, "/etc/apache2/sites-enabled/%s" % main_domain)
restart_apache()
def deploy():
"""
Push code, sync, migrate, generate media, restart
"""
assert_git_valid()
with cd(env.root_dir):
with prefix(env.activate):
# clone the latest repo
if file_exists("_latest"):
abort("deploy halted: remove dead _latest clone")
run("git clone -b %(branch)s %(repo)s releases/_latest" % env)
# link settings_local.py
# TODO: handle template
file_link("/srv/%(app)s/shared/settings_local.py", "/srv/%(app)s/releases/_latest/%(app)s/settings_local.py" % env)
# get the latest release
env.latest_release = run("git --git-dir=releases/_latest/.git rev-parse origin/%(branch)s" % env)
# pip install requirements
output = run("pip install -r releases/_latest/requirements.txt")
if output.failed:
abort('deploy halted: pip install failed!')
# migrate
output = run("python releases/_latest/%(app)s/manage.py migrate" % env)
if output.failed:
abort('deploy halted: migration failed!')
# collectstatic
file_link("/srv/%(app)s/shared/%(static_root)s" % env, "releases/_latest/%(app)s/%(static_root)s" % env)
output = run('python releases/_latest/%(app)s/manage.py collectstatic --noinput -i "*.pyc"' % env)
if output.failed:
abort('deploy halted: collectstatic failed!')
# swap symlinks
run("mv releases/_latest releases/%(latest_release)s" % env)
if file_exists("current"):
if file_exists("previous"):
run("rm previous")
run("mv current previous")
# link current to latest release
file_link("/srv/%(app)s/releases/%(latest_release)s" % env, "current")
# restart supervisord
# TODO: there must be a better way
#sudo('supervisorctl status %(app)s | sed "s/.*[pid ]\([0-9]\+\)\,.*/\\1/" | xargs kill -HUP' % env)
# restart gunicorn
sudo('/etc/init.d/%(app)s restart')
os.mkdir("/root/.unison")
sed_cmd3 = 'sed "s/SLAVESERVER/'+slaveip+'/g" '+installation_path+'/conf/default.prf > /root/.unison/default.prf'
subprocess.call(sed_cmd3, shell=True)
rsync_cmd1 = 'rsync -av '+installation_path+'/conf/lsyncd_master.conf /etc/lsyncd.conf'
subprocess.call(rsync_cmd1, shell=True)
sed_cmd4 = 'sed -e "s/MASTERSERVER/'+masteripalone+'/g" -e "s/MASTERSSHPORT/'+masterport+'/g" '+installation_path+'/conf/lsyncd_slave.conf > /tmp/nDeploy_lsyncd.conf'
subprocess.call(sed_cmd4, shell=True)
cuisine.rsync("/tmp/nDeploy_lsyncd.conf", "/etc/lsyncd.conf")
os.remove("/tmp/nDeploy_lsyncd.conf")
cuisine.run('sed -i "s/^UMASK/#UMASK/" /etc/login.defs')
if not os.path.isdir("/etc/nginx/"+slaveserver):
os.mkdir("/etc/nginx/"+slaveserver)
cuisine.dir_ensure("/etc/nginx/"+slaveserver)
if not cuisine.file_is_link("/etc/nginx/sites-enabled"):
cuisine.dir_remove("/etc/nginx/sites-enabled")
cuisine.file_link("/etc/nginx/"+slaveserver, "/etc/nginx/sites-enabled", symbolic=True)
subprocess.call('systemctl enable csync2.socket', shell=True)
subprocess.call('systemctl start csync2.socket', shell=True)
cuisine.run('systemctl enable csync2.socket')
cuisine.run('systemctl start csync2.socket')
# Do a Manual csync2 sync
subprocess.call("/usr/sbin/csync2 -xv", shell=True)
subprocess.call('grep "/usr/sbin/csync2" /etc/crontab || echo "* * * * * root /usr/sbin/csync2 -x" >> /etc/crontab', shell=True)
subprocess.call('systemctl restart crond.service', shell=True)
subprocess.call('systemctl enable lsyncd.service', shell=True)
subprocess.call('systemctl start lsyncd.service', shell=True)
cuisine.run('systemctl enable lsyncd.service')
cuisine.run('systemctl start lsyncd.service')
cuisine.run('systemctl enable nginx.service')
cuisine.run('systemctl start nginx.service')
# Creating the cluster config file
def link_django_settings():
""" links the django settings file for the current env """
with cd("/opt/rtd/apps/readthedocs/current/readthedocs.org/readthedocs/settings"
):
if not file_exists("currentenv.py"):
file_link("prod.py", "currentenv.py")
sed_cmd3 = 'sed "s/SLAVESERVER/' + slaveip + '/g" ' + installation_path + '/conf/default.prf > /root/.unison/default.prf'
subprocess.call(sed_cmd3, shell=True)
rsync_cmd1 = 'rsync -av ' + installation_path + '/conf/lsyncd_master.conf /etc/lsyncd.conf'
subprocess.call(rsync_cmd1, shell=True)
sed_cmd4 = 'sed -e "s/MASTERSERVER/' + masteripalone + '/g" -e "s/MASTERSSHPORT/' + masterport + '/g" ' + installation_path + '/conf/lsyncd_slave.conf > /tmp/nDeploy_lsyncd.conf'
subprocess.call(sed_cmd4, shell=True)
cuisine.rsync("/tmp/nDeploy_lsyncd.conf", "/etc/lsyncd.conf")
os.remove("/tmp/nDeploy_lsyncd.conf")
cuisine.run('sed -i "s/^UMASK/#UMASK/" /etc/login.defs')
if not os.path.isdir("/etc/nginx/" + slaveserver):
os.mkdir("/etc/nginx/" + slaveserver)
cuisine.dir_ensure("/etc/nginx/" + slaveserver)
if not cuisine.file_is_link("/etc/nginx/sites-enabled"):
cuisine.dir_remove("/etc/nginx/sites-enabled")
cuisine.file_link("/etc/nginx/" + slaveserver,
"/etc/nginx/sites-enabled",
symbolic=True)
subprocess.call('systemctl enable csync2.socket', shell=True)
subprocess.call('systemctl start csync2.socket', shell=True)
cuisine.run('systemctl enable csync2.socket')
cuisine.run('systemctl start csync2.socket')
# Do a Manual csync2 sync
subprocess.call("/usr/sbin/csync2 -xv", shell=True)
subprocess.call(
'grep "/usr/sbin/csync2" /etc/crontab || echo "* * * * * root /usr/sbin/csync2 -x" >> /etc/crontab',
shell=True)
subprocess.call('systemctl restart crond.service', shell=True)
subprocess.call('systemctl enable lsyncd.service', shell=True)
subprocess.call('systemctl start lsyncd.service', shell=True)
cuisine.run('systemctl enable lsyncd.service')
cuisine.run('systemctl start lsyncd.service')
def symlinks():
with fabtools.python.virtualenv(VIRTUAL_ENV):
with cd(os.path.join(WWW_DIR, "DjangoSite")):
file_link("settings.prod.py", "settings.py")
def link_django_settings():
""" links the django settings file for the current env """
with cd("/opt/rtd/apps/readthedocs/current/readthedocs.org/readthedocs/settings"):
if not file_exists("currentenv.py"):
file_link("prod.py","currentenv.py")
def symlinks():
with fabtools.python.virtualenv(VIRTUAL_ENV):
with cd(os.path.join(WWW_DIR, "DjangoSite")):
file_link("settings.prod.py", "settings.py")