def install_virtualenv():
"""
Create a new virtual environment for a project.
"""
require_env('virtualenv_path', 'project_path')
# Replaces fabtools version since ther is some bug with pip install script
# looks like curl fails on redirect.
fabtools.deb.install('python-pip')
# fabtools.python.install_pip()
fabtools.python.install('virtualenv', use_sudo=True)
if exists(env.virtualenv_path):
run('rm -r %s' % env.virtualenv_path)
# Create virtualenv
run('virtualenv --no-site-packages -p %s --distribute %s' %
(env.python, env.virtualenv_path))
if not exists('~/activate'):
run('ln -s %s/bin/activate ~/activate' % env.virtualenv_path)
with cd(env.virtualenv_path):
append_extract = 'export PYTHONPATH=%s' % env.project_path
sudo('echo "%s" >> bin/activate' % append_extract)
# Make shortcut
sudo('ln -sf %s %s' % (env.virtualenv_path, os.path.join(env.project_path, 'venv')))
_print(green('virtualenv installed'))
# Make shortcut
sudo('ln -sf %s %s' % (env.virtualenv_path, os.path.join(env.project_path, 'venv')))
def api_path():
"""Runs commands within the project's directory."""
require_env('project_path', 'virtualenv_path')
with fabtools.python.virtualenv(env.virtualenv_path):
with cd(env.django_path):
yield
def configure_supervisor():
"""Configure supervisor processes"""
SUPERVISOR_TASKS = dict(
django={},
scrapers={}
)
require_env('project_path', 'virtualenv_path', 'frontend_ip',
'django_settings', 'project_user')
SUPERVISOR_TASKS['django']['command'] = (
'/home/vagrant/smartnews/venv/bin/uwsgi --module %s.wsgi --socket %s --chmod-socket=666'
% (env.project_wsgi, env.uwsgi_socket))
SUPERVISOR_TASKS['django']['directory'] = env.project_path
SUPERVISOR_TASKS['scrapers']['command'] = (
'%s/bin/run_celery' % env.project_path
)
SUPERVISOR_TASKS['scrapers']['directory'] = env.project_path
for process_name, options in SUPERVISOR_TASKS.iteritems():
kwargs = dict(
directory=options['directory'],
environment='PYTHONPATH="%s"' % os.path.join(env.project_path, 'django_app'),
autostart='true',
autorestart='false',
master='true',
startsecs=3,
user=env.supervisor_user,
loglevel='debug',
virtualenv='/home/vagrant/smartnews/venv/',
)
options['command'] = options['command'].format(env.virtualenv_path)
process_kwargs = kwargs.copy()
process_kwargs.update(
stdout_logfile=LOG_PATH % process_name,
**options
)
sudo('touch /var/log/%s.log' % (process_name))
fabtools.require.file('/var/log/%s.log' % (process_name))
sudo('chown %s:%s /var/log/%s.log' % (env.supervisor_user,
env.supervisor_user,
process_name))
for key in process_kwargs:
if callable(process_kwargs[key]):
process_kwargs[key] = process_kwargs[key]()
fabtools.require.supervisor.process(process_name, **process_kwargs)
# autostart: attached supervisord for /etc/init.d/ if needed
if not exists('/etc/init.d/supervisor', use_sudo=True):
sudo('update-rc.d supervisor defaults')
def install_debian_cassandra():
# debian_cassandra has all the requirements to install cassandra
require_env('cassandra_packages')
cassandra_requirements = os.path.join(os.path.dirname(__file__),
env.cassandra_packages)
with open(cassandra_requirements, 'r') as file_:
packages = [line.strip() for line in file_]
cassandra_sources()
fabtools.deb.install(packages)
fabtools.deb.upgrade()
def install_debian_packages():
require_env('debian_packages')
fabtools.deb.update_index()
filename = os.path.join(os.path.dirname(__file__),
env.debian_packages)
with open(filename, 'r') as file_:
packages = [line.strip() for line in file_]
fabtools.deb.install(packages)
fabtools.deb.upgrade()
def install_git_repository(branch=None):
require_env('project_path', 'git_url')
if not exists(env.project_path):
if env.get('is_vagrant') and exists('/vagrant'):
run('ln -s /vagrant %s' % env.project_path)
else:
# not tested on testing/staging
fabtools.require.directory(env.project_path)
fabtools.git.clone(env.git_url, path=env.project_path)
if branch:
fabtools.git.checkout(env.project_path, branch=branch)
def install_rvm():
"""
Install rvm
"""
require_env('ruby_version')
run('gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3')
run('curl -L https://get.rvm.io | bash -s stable')
if not exists('~/.rvm/'):
run('ln -sd /usr/local/rvm ~/.rvm')
run('source ~/.rvm/scripts/rvm')
run('rvm install %s' % env.ruby_version)
def install_nodeenv():
"""
Install nodeenv
"""
require_env('nodeenv_path')
nodejs_path = '/usr/bin/nodejs'
node_path = '/usr/bin/node'
# if not exists(node_path):
# sudo('ln -s %s %s' % (nodejs_path, node_path))
if exists(env.nodeenv_path):
run('rm -r %s' % env.nodeenv_path)
with project():
run('nodeenv --node=system %s' % env.nodeenv_path)
def configure_nginx():
"""Configure nginx"""
require_env('django_static_url', 'django_static_root', 'cdn_static_root',
'user_site_static_root', 'uwsgi_socket',
'uwsgi_socket_web')
django_config = """
upstream django {
server unix://%(uwsgi_socket)s;
}
upstream web {
server unix://%(uwsgi_socket_web)s;
}
server {
listen %(port)s;
server_name %(server_name)s;
client_max_body_size 50M;
access_log /var/log/nginx/%(server_name)s.log;
error_log /var/log/nginx/%(server_name)s_error.log;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
location /static/ {
alias %(django_static_root)s;
sendfile off;
}
location / {
uwsgi_pass django;
include uwsgi_params;
}
location /web/ {
rewrite /web/(.+) /$1 break;
uwsgi_pass web;
include uwsgi_params;
}
}
"""
static_config = """
server {
listen %(port)s;
server_name %(server_name)s;
access_log /var/log/nginx/%(server_name)s.log;
error_log /var/log/nginx/%(server_name)s_error.log;
location / {
alias %(static_root)s;
sendfile off;
}
}
"""
live_config = """
server {
listen %(port)s;
server_name %(server_name)s;
access_log /var/log/nginx/%(server_name)s.log;
error_log /var/log/nginx/%(server_name)s_error.log;
location %(location)s {
proxy_pass http://127.0.0.1:9000/;
}
}
"""
django_kwargs = dict(
port=80,
django_static_url=env.django_static_url,
django_static_root=env.django_static_root,
uwsgi_socket=env.uwsgi_socket,
uwsgi_socket_web=env.uwsgi_socket_web,
)
cdn_kwargs = dict(
port=80,
static_root=env.cdn_static_root
)
user_site_kwargs = dict(
port=80,
static_root=env.user_site_static_root,
)
cdn_live_kwargs = dict(
port=80,
location='/widget/',
)
user_site_live_kwargs = dict(
port=80,
location='/',
)
fabtools.require.nginx.server()
fabtools.require.nginx.disable('default')
fabtools.require.nginx.site('jbg.dev', django_config, **django_kwargs)
fabtools.require.nginx.site('cdn.jbg.dev', static_config, **cdn_kwargs)
fabtools.require.nginx.site('jbg.usersite.dev', static_config,
**user_site_kwargs)
fabtools.require.nginx.site('cdn.jbg.live', live_config,
**cdn_live_kwargs)
fabtools.require.nginx.site('jbg.usersite.live', live_config,
**user_site_live_kwargs)
def nodeenv():
"""Runs commands within nodeenv."""
require_env('nodeenv_path')
with nested(prefix("source %s/bin/activate" % env.nodeenv_path)):
yield
def rvm():
"""Runs commands within rvm."""
require_env('ruby_version')
with nested(prefix("source ~/.rvm/scripts/rvm"),
prefix('rvm %s' % env.ruby_version)):
yield
