def mysql():
"""
Setup MySQL server, user and database
"""
from fabtools import require
import fabtools
require.mysql.server(password='******')
with settings(mysql_user='******', mysql_password='******'):
fabtools.mysql.create_user('bob', 'password', host='host1')
fabtools.mysql.create_user('bob', 'password', host='host2')
assert fabtools.mysql.user_exists('bob', host='host1')
assert fabtools.mysql.user_exists('bob', host='host2')
assert not fabtools.mysql.user_exists('bob', host='localhost')
require.mysql.user('myuser', 'foo')
assert fabtools.mysql.user_exists('myuser')
require.mysql.database('mydb', owner='myuser')
assert fabtools.mysql.database_exists('mydb')
# Test that we can run queries as user foo
with settings(mysql_user='******', mysql_password='******'):
fabtools.mysql.query('select 1;')
# Test that we can run queries without supplying the password
require.file('.my.cnf', contents="[mysql]\npassword=foo")
with settings(mysql_user='******'):
fabtools.mysql.query('select 2;')
def deploy_config():
"""Deploy config file to Flask instance folder."""
require.files.directory('%s/instance' % env.app_dir, use_sudo=True)
require.file('%(app_dir)s/instance/config.py' % env,
source='%(config_file)s' % env,
use_sudo=True,
group='www-data')
def process(name, **kwargs):
"""
Require a supervisor process
"""
from fabtools import require
require.deb.package('supervisor')
require.service.started('supervisor')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
with watch(filename, True, reload_config):
require.file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
def upload_sqldump(app):
if app.dbdump:
if re.match('http(s)?://', app.dbdump):
fname = 'dump.sql.gz'
url = app.dbdump
auth = ''
else:
latest = cdstar.get_latest_bitstream(app.dbdump)
fname, url = latest.name, latest.url
auth = '-u"{0}:{1}" '.format(os.environ['CDSTAR_USER_BACKUP'], os.environ['CDSTAR_PWD_BACKUP'])
target = pathlib.PurePosixPath('/tmp') / fname
run('curl -s -o {0} {1} {2}'.format(target, auth, url))
else:
db_name = prompt('Replace with dump of local database:', default=app.name)
sqldump = pathlib.Path(tempfile.mktemp(suffix='.sql.gz', prefix='%s-' % db_name))
target = pathlib.PurePosixPath('/tmp') / sqldump.name
db_user = '******' if PLATFORM == 'windows' else ''
local('pg_dump %s--no-owner --no-acl -Z 9 -f %s %s' % (db_user, sqldump, db_name))
require.file(str(target), source=str(sqldump))
sqldump.unlink()
if app.stack == 'soundcomparisons':
sudo('echo "drop database {0};" | mysql'.format(app.name))
require.mysql.database(app.name, owner=app.name)
sudo('gunzip -c {0} | mysql -u {1} --password={1} -D {1}'.format(target, app.name), user=app.name)
else:
# TODO: assert supervisor.process_status(app.name) != 'RUNNING'
if postgres.database_exists(app.name):
require_postgres(app, drop=True)
sudo('gunzip -c %s | psql -d %s' % (target, app.name), user=app.name)
sudo('vacuumdb -zf %s' % app.name, user='******')
files.remove(str(target))
def headless():
# How do I enable the "multiverse" repository?
# http://askubuntu.com/questions/89096/how-do-i-enable-the-multiverse-repository
sudo('sed -i "/^# deb.*multiverse/ s/^# //" /etc/apt/sources.list')
timezone()
execute(ntpd)
require_deb_packages( '''
sudo
screen
htop
nmap
iotop
mc
ssh
nano
cpuid
curl
wget
nfs-kernel-server
nfs-common
''')
require.directory('bin')
require.file('.profile', contents=cfg('profile.conf'))
def require_postgres(app, drop=False):
if drop:
with cd('/var/lib/postgresql'):
sudo('dropdb %s' % app.name, user='******')
with shell_env(SYSTEMD_PAGER=''):
require.postgres.server()
require.postgres.user(app.name, password=app.name)
require.postgres.database(app.name, owner=app.name)
if app.pg_unaccent:
sql = 'CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;'
sudo('psql -c "%s" -d %s' % (sql, app.name), user='******')
if app.pg_collkey:
pg_dir, = run('find /usr/lib/postgresql/ -mindepth 1 -maxdepth 1 -type d').splitlines()
pg_version = pathlib.PurePosixPath(pg_dir).name
if not exists('/usr/lib/postgresql/%s/lib/collkey_icu.so' % pg_version):
require.deb.packages(['postgresql-server-dev-%s' % pg_version, 'libicu-dev'])
with cd('/tmp'):
sudo_upload_template('pg_collkey.Makefile', dest='Makefile', pg_version=pg_version)
require.file('collkey_icu.c', source=str(PG_COLLKEY_DIR / 'collkey_icu.c'))
run('make')
sudo('make install')
with cd('/tmp'):
require.file('collkey_icu.sql', source=str(PG_COLLKEY_DIR / 'collkey_icu.sql'))
sudo('psql -f collkey_icu.sql -d %s' % app.name, user='******')
def setup_s3cmd(aws_access_key_id, aws_secret_access_key, s3cfg=None):
"""
Install s3cmd for multipart uploads to Amazon S3
"""
s3cfg = s3cfg if s3cfg is not None else DEFAULT_S3_CFG
# Install s3cmd
if not is_file('/usr/local/bin/s3cmd'):
require.file(url='http://sourceforge.net/projects/s3tools/files/s3cmd/1.5.0-alpha1/s3cmd-1.5.0-alpha1.tar.gz')
if not is_dir('s3cmd-1.5.0-alpha1'):
run('tar xzf s3cmd-1.5.0-alpha1.tar.gz')
with cd('s3cmd-1.5.0-alpha1'):
require.deb.package('python-setuptools')
sudo('python setup.py install')
# Optional dependencies
require.python.package('python-magic', use_sudo=True)
# S3 config file (including credentials)
require.file(
'/var/lib/postgresql/.s3cfg',
contents=s3cfg.format(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key),
use_sudo=True
)
def installed_from_source(version=VERSION):
"""
Require Redis to be installed from source.
The compiled binaries will be installed in ``/opt/redis-{version}/``.
"""
from fabtools import require
require.user('redis', home='/var/lib/redis')
dest_dir = '/opt/redis-%(version)s' % locals()
require.directory(dest_dir, use_sudo=True, owner='redis')
if not is_file('%(dest_dir)s/redis-server' % locals()):
with cd('/tmp'):
# Download and unpack the tarball
tarball = 'redis-%(version)s.tar.gz' % locals()
require.file(tarball, url='http://redis.googlecode.com/files/' + tarball)
run('tar xzf %(tarball)s' % locals())
# Compile and install binaries
require.deb.package('build-essential')
with cd('redis-%(version)s' % locals()):
run('make')
for filename in BINARIES:
sudo('cp -pf src/%(filename)s %(dest_dir)s/' % locals())
sudo('chown redis: %(dest_dir)s/%(filename)s' % locals())
def livebuild(proxy):
print green("Livebuild with " + proxy)
pkg('openssh-server live-build python git-core zsh',
'debootstrap')
require.directory('/var/build/')
mount_line('tmpfs /var/tmp tmpfs defaults 0 0')
run_as_root('cat /etc/fstab')
require.file('/etc/http_proxy', contents="http://%s/" % proxy,
owner='root', group='root', mode='644',
verify_remote=True, use_sudo=True)
require.file('/etc/profile.d/proxy.sh',
contents='export http_proxy="http://%s/"' % proxy,
owner='root', group='root', mode='644',
verify_remote=True, use_sudo=True)
require.file('/usr/local/bin/manualbuild.sh',
source='files/bin/manualbuild.sh', owner='root', group='root',
mode='755',
verify_remote=True, use_sudo=True)
require.file('/usr/local/bin/gitbuild.sh',
source='files/bin/gitbuild.sh', owner='root', group='root',
mode='755',
verify_remote=True, use_sudo=True)
require.file('/etc/cron.daily/cacherepo',
source='files/bin/cron_cacherepo', owner='root', group='root',
mode='755',
verify_remote=True, use_sudo=True)
def uninstall_couchdb():
'''
Uninstall CouchDB 1.3.0 or 1.5.0
'''
version = get_couchdb_version()
require.file(
url='http://apache.crihan.fr/dist/couchdb/source/' +
'%s/apache-couchdb-%s.tar.gz' % (version, version))
run('tar -xzvf apache-couchdb-%s.tar.gz' % version)
with cd('apache-couchdb-%s' % version):
sudo('./configure')
sudo('make uninstall')
sudo('make distclean')
su_delete('/usr/local/share/couchdb')
su_delete('/usr/local/lib/couchdb')
su_delete('/usr/local/var/lib/couchdb')
su_delete('/usr/local/var/log/couchdb')
su_delete('/usr/local/var/run/couchdb')
su_delete('/usr/local/share/doc/couchdb')
su_delete('/usr/local/bin/couchjs')
su_delete('/usr/local/bin/couchdb')
su_delete('apache-couchdb-%s' % version)
su_delete('apache-couchdb-%s.tar.gz' % version)
su_delete('/etc/supervisor/conf.d/couchdb.conf')
su_delete('/etc/cozy/couchdb.login')
supervisor.update_config()
print(green('CouchDB %s successfully uninstalled' % version))
def install_node10():
'''
install node 0.10.26
'''
if is_pi():
with settings(warn_only=True):
result = run('node -v')
is_installed = result.find('v0.10.26')
if is_installed != -1:
print(green("Node.js is already installed"))
return True
version = '0.10.26'
node_url = 'http://nodejs.org/dist/v{0}/node-v{0}-linux-arm-pi.tar.gz'
require.file(url=node_url.format(version))
run('tar -xzvf node-v%s-linux-arm-pi.tar.gz' % version)
delete_if_exists('/opt/node')
require.directory('/opt/node', owner='root')
sudo('mv node-v%s-linux-arm-pi/* /opt/node' % version)
sudo('ln -s /opt/node/bin/node /usr/local/bin/node')
sudo('ln -s /opt/node/bin/node /usr/bin/node')
sudo('ln -s /opt/node/bin/npm /usr/local/bin/npm')
sudo('ln -s /opt/node/bin/npm /usr/bin/npm')
else:
require.nodejs.installed_from_source('0.10.26')
print(green('node 0.10.26 successfully installed'))
def install_nodejs():
"""
Test low level API
"""
from fabtools import nodejs
from fabtools import require
from fabtools.files import is_file
# Upload local copy of source archive to speed up tests
local_cache = '~/.vagrant.d/cache/fabtools/node-v%s.tar.gz' % nodejs.DEFAULT_VERSION
if os.path.exists(local_cache):
put(local_cache)
# Install Node.js from source
if nodejs.version() != nodejs.DEFAULT_VERSION:
nodejs.install_from_source()
assert is_file('/usr/local/bin/node')
assert nodejs.version() == nodejs.DEFAULT_VERSION
# Install / uninstall global package
if not nodejs.package_version('underscore'):
nodejs.install_package('underscore', version='1.4.2')
assert nodejs.package_version('underscore') == '1.4.2'
assert is_file('/usr/local/lib/node_modules/underscore/underscore.js')
nodejs.uninstall_package('underscore')
assert nodejs.package_version('underscore') == None
assert not is_file('/usr/local/lib/node_modules/underscore/underscore.js')
# Install / uninstall local package
if not nodejs.package_version('underscore', local=True):
nodejs.install_package('underscore', version='1.4.2', local=True)
assert is_file('node_modules/underscore/underscore.js')
assert nodejs.package_version('underscore', local=True) == '1.4.2'
nodejs.uninstall_package('underscore', local=True)
assert nodejs.package_version('underscore', local=True) == None
assert not is_file('node_modules/underscore/underscore.js')
# Install dependencies from package.json file
require.directory('nodetest')
with cd('nodetest'):
require.file('package.json', contents=json.dumps({
'name': 'nodetest',
'version': '1.0.0',
'dependencies': {
'underscore': '1.4.2'
}
}))
nodejs.install_dependencies()
assert is_file('node_modules/underscore/underscore.js')
assert nodejs.package_version('underscore', local=True) == '1.4.2'
def install_node10():
'''
install node 0.10.26
'''
if is_arm():
with settings(warn_only=True):
result = run('node -v')
is_installed = result.find('v0.10.26')
if is_installed != -1:
print(green("Node.js is already installed"))
return True
version = '0.10.26'
node_url = 'http://nodejs.org/dist/v{0}/node-v{0}-linux-arm-pi.tar.gz'
require.file(url=node_url.format(version))
run('tar -xzvf node-v%s-linux-arm-pi.tar.gz' % version)
delete_if_exists('/opt/node')
require.directory('/opt/node', use_sudo=True, owner='root')
sudo('mv node-v%s-linux-arm-pi/* /opt/node' % version)
sudo('ln -s /opt/node/bin/node /usr/local/bin/node')
sudo('ln -s /opt/node/bin/node /usr/bin/node')
sudo('ln -s /opt/node/bin/npm /usr/local/bin/npm')
sudo('ln -s /opt/node/bin/npm /usr/bin/npm')
else:
require.nodejs.installed_from_source('0.10.26')
print(green('node 0.10.26 successfully installed'))
def uninstall_couchdb():
'''
Uninstall CouchDB 1.3.0 or 1.5.0
'''
version = get_couchdb_version()
require.file(url='http://apache.crihan.fr/dist/couchdb/source/' +
'%s/apache-couchdb-%s.tar.gz' % (version, version))
run('tar -xzvf apache-couchdb-%s.tar.gz' % version)
with cd('apache-couchdb-%s' % version):
sudo('./configure')
sudo('make uninstall')
sudo('make distclean')
su_delete('/usr/local/share/couchdb')
su_delete('/usr/local/lib/couchdb')
su_delete('/usr/local/var/lib/couchdb')
su_delete('/usr/local/var/log/couchdb')
su_delete('/usr/local/var/run/couchdb')
su_delete('/usr/local/share/doc/couchdb')
su_delete('/usr/local/bin/couchjs')
su_delete('/usr/local/bin/couchdb')
su_delete('apache-couchdb-%s' % version)
su_delete('apache-couchdb-%s.tar.gz' % version)
su_delete('/etc/supervisor/conf.d/couchdb.conf')
su_delete('/etc/cozy/couchdb.login')
# removes dependency of CouchDB
require.deb.uninstall('erlang', False, ['--auto-remove'])
supervisor.update_config()
print(green('CouchDB %s successfully uninstalled' % version))
def load_sqldump(app, url=DUMP_URL, md5=DUMP_MD5):
if console.confirm('Fill the database with %s?' % url, default=False):
_, _, filename = url.rpartition('/')
with cd('/tmp'):
require.file(filename, url=url, md5=md5)
sudo('gunzip -c %s | psql %s' % (filename, app.name),
user=app.name)
def devel():
"""
Developer tools installation
"""
#TODO: gitconfig
deb.install(['build-essential', 'git'])
require.file('/etc/gitconfig', source='files/gitconfig', use_sudo=True)
def instance(name, version=VERSION, **kwargs):
"""
Require a Redis instance to be running
The instance will be managed using supervisord.
"""
from fabtools import require
installed_from_source(version)
require.directory('/etc/redis', use_sudo=True, owner='redis')
require.directory('/var/db/redis', use_sudo=True, owner='redis')
require.directory('/var/log/redis', use_sudo=True, owner='redis')
require.directory('/var/run/redis', use_sudo=True, owner='redis')
# Required for background saving
with settings(warn_only=True):
require.system.sysctl('vm.overcommit_memory', '1')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('bind', '127.0.0.1')
params.setdefault('port', '6379')
params.setdefault('logfile', '/var/log/redis/redis-%(name)s.log' % locals())
params.setdefault('loglevel', 'verbose')
params.setdefault('dbfilename', '/var/db/redis/redis-%(name)s-dump.rdb' % locals())
params.setdefault('save', ['900 1', '300 10', '60 10000'])
# Build config file from parameters
# (keys such as 'save' may result in multiple config lines)
lines = []
for key, value in sorted(params.items()):
if isinstance(value, list):
for elem in value:
lines.append("%s %s" % (key, elem))
else:
lines.append("%s %s" % (key, value))
redis_server = '/opt/redis-%(version)s/redis-server' % locals()
config_filename = '/etc/redis/%(name)s.conf' % locals()
# Upload config file
context = dict(need_restart=False)
def on_change():
context['need_restart'] = True
with watch(config_filename, True, on_change):
require.file(config_filename, contents='\n'.join(lines),
use_sudo=True, owner='redis')
# Use supervisord to manage process
process_name = 'redis_%s' % name
require.supervisor.process(process_name,
user='******',
directory='/var/run/redis',
command="%(redis_server)s %(config_filename)s" % locals())
if context['need_restart']:
fabtools.supervisor.restart_process(process_name)
def build_box(requirements, settings):
"""
Args:
requirements (Requirements):
settings (Settings):
Yields:
BuildBoxEnv:
"""
local('vagrant destroy -f')
local('vagrant up')
with vagrant_settings():
require.deb.ppa('ppa:spotify-jyrki/dh-virtualenv', auto_accept=True)
require.deb.package(requirements.dh_virtualenv)
require.deb.packages(requirements.project_package)
require.directory('~/.ssh', mode='700')
require.file('~/.ssh/config', contents=settings.ssh_config)
gitconfig = os.path.expanduser('~/.gitconfig')
if os.path.exists(gitconfig):
require.file('~/.gitconfig', source=gitconfig)
require.git.working_copy(settings.repo, branch=settings.branch,
path=settings.project)
yield BuildBoxEnv(Settings.project)
local('vagrant destroy -f')
def setup_web():
require.deb.packages(['git', 'npm', 'nodejs-legacy', 'python3-dev', 'libxml2-dev', 'libxslt1-dev', 'libpq-dev'])
require.nginx.server()
require.nginx.site(env.server_name, template_source='nginx-site',
port=80,
server_alias='',
static_path=env.static_path)
# require.nginx.disabled('default')
with settings(warn_only=True):
if run('type bower').return_code:
sudo('npm install -g bower')
update_source()
require.directory(env.home + '/logs')
require.python.packages(['uwsgi', 'virtualenv', 'ipython', 'celery'], use_sudo=True)
require.file('/etc/init/uwsgi.conf', source='uwsgi.conf', use_sudo=True)
# require.file('/etc/init/celery.conf', source='celery.conf', use_sudo=True)
require.directory('/etc/uwsgi', use_sudo=True)
require.files.template_file('/etc/uwsgi/%s.ini' % env.project_name,
template_source='uwsgi.ini',
context=env, use_sudo=True)
require.service.started('uwsgi')
require.directory(env.home + '/bin')
require.files.template_file(env.home + '/bin/djangorc',
template_source='bin/djangorc',
context=env)
require.files.template_file(env.home + '/bin/loadenv',
template_source='bin/loadenv',
context=env)
run('chmod +x ' + env.home + '/bin/loadenv')
def installed_from_source(version=VERSION):
"""
Require Redis to be installed from source
"""
from fabtools import require
require.user('redis')
dest_dir = '/opt/redis-%(version)s' % locals()
require.directory(dest_dir, use_sudo=True, owner='redis')
if not is_file('%(dest_dir)s/redis-server' % locals()):
with cd('/tmp'):
# Download and unpack the tarball
tarball = 'redis-%(version)s.tar.gz' % locals()
require.file(tarball,
url='http://redis.googlecode.com/files/' + tarball)
run('tar xzf %(tarball)s' % locals())
# Compile and install binaries
require.deb.package('build-essential')
with cd('redis-%(version)s' % locals()):
run('make')
for filename in BINARIES:
sudo('cp -pf src/%(filename)s %(dest_dir)s/' % locals())
sudo('chown redis: %(dest_dir)s/%(filename)s' % locals())
def test_default_temp_dir(self, is_file, md5sum, put, umask, owner, mode):
owner.return_value = 'root'
umask.return_value = '0002'
mode.return_value = '0664'
from fabtools import require
require.file('/var/tmp/foo', source=__file__, use_sudo=True)
put.assert_called_with(__file__, '/var/tmp/foo', use_sudo=True, temp_dir='/tmp')
def process(name, **kwargs):
"""
Require a supervisor process
"""
from fabtools import require
require.deb.package("supervisor")
require.service.started("supervisor")
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault("autorestart", "true")
params.setdefault("redirect_stderr", "true")
# Build config file from parameters
lines = []
lines.append("[program:%(name)s]" % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = "/etc/supervisor/conf.d/%(name)s.conf" % locals()
with watch(filename, True, reload_config):
require.file(filename, contents="\n".join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == "STOPPED":
start_process(name)
def process(name, **kwargs):
"""
Require a supervisor process
"""
from fabtools import require
require.deb.package('supervisor')
require.service.started('supervisor')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
with watch(filename, callback=update_config, use_sudo=True):
require.file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
def process(name, **kwargs):
"""
Require a supervisor process to be running.
Keyword arguments will be used to build the program configuration
file. Some useful arguments are:
- ``command``: complete command including arguments (**required**)
- ``directory``: absolute path to the working directory
- ``user``: run the process as this user
- ``stdout_logfile``: absolute path to the log file
You should refer to the `supervisord documentation`_ for the
complete list of allowed arguments.
.. note:: the default values for the following arguments differs from
the supervisord defaults:
- ``autorestart``: defaults to ``true``
- ``redirect_stderr``: defaults to ``true``
Example::
from fabtools import require
require.supervisor.process('myapp',
command='/path/to/venv/bin/myapp --config production.ini --someflag',
directory='/path/to/working/dir',
user='******',
stdout_logfile='/path/to/logs/myapp.log',
)
.. _supervisord documentation: http://supervisord.org/configuration.html#program-x-section-values
"""
from fabtools import require
require.deb.package("supervisor")
require.service.started("supervisor")
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault("autorestart", "true")
params.setdefault("redirect_stderr", "true")
# Build config file from parameters
lines = []
lines.append("[program:%(name)s]" % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = "/etc/supervisor/conf.d/%(name)s.conf" % locals()
with watch(filename, callback=update_config, use_sudo=True):
require.file(filename, contents="\n".join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == "STOPPED":
start_process(name)
def ntpd():
require_deb_packages(
'''
ntp
ntpdate
'''
)
require.file('/etc/ntp.conf', contents=cfg('ntp.conf'), use_sudo=True)
def copy_archive(app, archive):
arc = 'archive.tgz'
with cd('/tmp'):
require.file(arc, source=archive)
sudo('tar -xzf {0}'.format(arc))
sudo('rm -rf {0}/files/*'.format(app.www_dir))
sudo('mv archive/* {0}/files'.format(app.www_dir))
sudo('rm {0}'.format(arc))
def base(proxy='127.0.0.1:3142'):
require.file("/etc/apt/apt.conf",
contents='Acquire::http { Proxy "http://%s"; };' % proxy)
keyrings()
devel()
if iputils.is_my_ip(proxy.split(':')[0]):
aptcacher()
livebuild(proxy)
def process(name, **kwargs):
"""
Require a supervisor process to be running.
Keyword arguments will be used to build the program configuration
file. Some useful arguments are:
- ``command``: complete command including arguments (**required**)
- ``directory``: absolute path to the working directory
- ``user``: run the process as this user
- ``stdout_logfile``: absolute path to the log file
You should refer to the `supervisor documentation`_ for the
complete list of allowed arguments.
.. note:: the default values for the following arguments differs from
the ``supervisor`` defaults:
- ``autorestart``: defaults to ``true``
- ``redirect_stderr``: defaults to ``true``
Example::
from fabtools import require
require.supervisor.process('myapp',
command='/path/to/venv/bin/myapp --config production.ini --someflag',
directory='/path/to/working/dir',
user='******',
stdout_logfile='/path/to/logs/myapp.log',
)
.. _supervisor documentation: http://supervisord.org/configuration.html#program-x-section-values
"""
from fabtools import require
require.deb.package('supervisor')
require.service.started('supervisor')
# Set default parameters
params = {}
params.update(kwargs)
params.setdefault('autorestart', 'true')
params.setdefault('redirect_stderr', 'true')
# Build config file from parameters
lines = []
lines.append('[program:%(name)s]' % locals())
for key, value in sorted(params.items()):
lines.append("%s=%s" % (key, value))
# Upload config file
filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals()
with watch(filename, callback=update_config, use_sudo=True):
require.file(filename, contents='\n'.join(lines), use_sudo=True)
# Start the process if needed
if process_status(name) == 'STOPPED':
start_process(name)
def install_couchdb():
'''
Install CouchDB 1.3.0 or 1.5.0 depending on the target distribution.
'''
# Check if controller is already installed, .
with settings(warn_only=True):
result = run('curl -X GET http://127.0.0.1:5984/')
is_installed = result.find('Welcome')
if is_installed != -1:
print(green("CouchDB is already installed"))
return True
packages = [
'erlang',
'libicu-dev',
'libcurl4-openssl-dev',
'curl'
]
if system.distrib_id() == 'Debian' \
and system.distrib_release().startswith('6'):
packages.append('libmozjs-dev')
else:
packages.append('libmozjs185-dev')
require.deb.packages(packages)
version = get_couchdb_version()
require.file(
url='http://apache.crihan.fr/dist/couchdb/source/' +
'%s/apache-couchdb-%s.tar.gz' % (version, version))
run('tar -xzvf apache-couchdb-%s.tar.gz' % version)
with cd('apache-couchdb-%s' % version):
run('./configure; make')
result = sudo('make install')
installed = result.find('You have installed Apache CouchDB,' +
' time to relax.')
if installed == -1:
print_failed('couchdb')
su_delete('apache-couchdb-%s' % version)
su_delete('apache-couchdb-%s.tar.gz' % version)
require.users.user('couchdb', home='/usr/local/var/lib/couchdb')
sudo('chown -R couchdb:couchdb /usr/local/etc/couchdb')
sudo('chown -R couchdb:couchdb /usr/local/var/lib/couchdb')
sudo('chown -R couchdb:couchdb /usr/local/var/log/couchdb')
sudo('chown -R couchdb:couchdb /usr/local/var/run/couchdb')
sudo('chmod 0770 /usr/local/etc/couchdb')
sudo('chmod 0770 /usr/local/var/lib/couchdb')
sudo('chmod 0770 /usr/local/var/log/couchdb')
sudo('chmod 0770 /usr/local/var/run/couchdb')
require.supervisor.process(
'couchdb', user='******',
command='couchdb', autostart='true',
environment='HOME=/usr/local/var/lib/couchdb')
print(green('CouchDB %s successfully installed' % version))
def install_couchdb():
'''
Install CouchDB 1.3.0 or 1.5.0 depending on the target distribution.
'''
# Check if controller is already installed, .
with settings(warn_only=True):
result = run('curl -X GET http://127.0.0.1:5984/')
is_installed = result.find('Welcome')
if is_installed != -1:
print(green("CouchDB is already installed"))
return True
packages = [
'erlang',
'libicu-dev',
'libcurl4-openssl-dev',
'curl'
]
if system.distrib_id() == 'Debian' \
and system.distrib_release().startswith('6'):
packages.append('libmozjs-dev')
else:
packages.append('libmozjs185-dev')
require.deb.packages(packages)
version = get_couchdb_version()
require.file(
url='http://apache.crihan.fr/dist/couchdb/source/' +
'%s/apache-couchdb-%s.tar.gz' % (version, version))
run('tar -xzvf apache-couchdb-%s.tar.gz' % version)
with cd('apache-couchdb-%s' % version):
run('./configure; make')
result = sudo('make install')
installed = result.find('You have installed Apache CouchDB,' +
' time to relax.')
if installed == -1:
print_failed('couchdb')
su_delete('apache-couchdb-%s' % version)
su_delete('apache-couchdb-%s.tar.gz' % version)
require.users.user('couchdb', home='/usr/local/var/lib/couchdb')
sudo('chown -R couchdb:couchdb /usr/local/etc/couchdb')
sudo('chown -R couchdb:couchdb /usr/local/var/lib/couchdb')
sudo('chown -R couchdb:couchdb /usr/local/var/log/couchdb')
sudo('chown -R couchdb:couchdb /usr/local/var/run/couchdb')
sudo('chmod 0770 /usr/local/etc/couchdb')
sudo('chmod 0770 /usr/local/var/lib/couchdb')
sudo('chmod 0770 /usr/local/var/log/couchdb')
sudo('chmod 0770 /usr/local/var/run/couchdb')
require.supervisor.process(
'couchdb', user='******',
command='couchdb', autostart='true',
environment='HOME=/usr/local/var/lib/couchdb')
print(green('CouchDB %s successfully installed' % version))
def service_add():
"""
Add the systemd service
"""
with cd('/lib/systemd/system'):
require.file('gage-logger.service',
source='gage-logger.service',
use_sudo=True)
sudo('systemctl enable gage-logger.service')
def provision_glassfish():
require.file('glassfish.zip', url='http://download.java.net/glassfish/3.1.2.2/release/glassfish-3.1.2.2.zip')
run('unzip -o glassfish.zip')
run('echo "AS_ADMIN_PASSWORD="******"AS_ADMIN_NEWPASSWORD=adminadmin" >> /tmp/password.txt')
run('glassfish3/bin/asadmin --user admin --passwordfile /tmp/password.txt change-admin-password --domain_name domain1')
run('echo "AS_ADMIN_PASSWORD=adminadmin" > /tmp/password.txt')
run('glassfish3/bin/asadmin start-domain && glassfish3/bin/asadmin --passwordfile /tmp/password.txt --host localhost --port 4848 enable-secure-admin && glassfish3/bin/asadmin stop-domain')
run('rm /tmp/password.txt')
def install_services():
install_redis()
run('rm -rf /tmp/redis')
require.file('/etc/redis.conf', source='files/redis.conf', use_sudo=True)
require.file('/etc/redis_cache.conf', source='files/redis_cache.conf', use_sudo=True)
if not files.exists('/var/run/supervisor.sock'):
sudo('supervisord -c /etc/supervisor/supervisord.conf')
supervisor.update_config()
install_fluentd()
def setup_postgresql_backups(instance_name, aws_s3_prefix, aws_access_key_id, aws_secret_access_key, backup_script=None, backup_config=None):
"""
Setup daily SQL dumps of the Postgres server to Amazon S3
"""
backup_script = backup_script if backup_script is not None else DEFAULT_BACKUP_SCRIPT
backup_config = backup_config if backup_config is not None else DEFAULT_BACKUP_CONFIG
puts(green('Setting up Postgresql backups'))
# Cleanup old config files
sudo('rm -f /etc/default/pg_backup')
sudo('rm -f /etc/cron.d/pg_backup')
# Skip if backups are not enabled
if not env.enable_backups:
puts('Backups not enabled, skipping...')
return
# Skip if backups are not configured
if not aws_s3_prefix:
puts('Target bucket not configured, skipping...')
return
setup_s3cmd(aws_access_key_id, aws_secret_access_key)
require.deb.package('tree')
# Postgres backup script
backup_script_path = '/usr/local/bin/pg_backup_rotated.sh'
config_name = 'pg_backup_{}'.format(instance_name)
require.file(
backup_script_path,
contents=backup_script.format(config_name=config_name),
owner='root',
group='root',
mode='0755',
use_sudo=True
)
# Postgres backup configuration file
require.file(
'/etc/default/{}'.format(config_name),
contents=backup_config.format(aws_s3_prefix=aws_s3_prefix),
owner='root',
group='root',
mode='0644',
use_sudo=True
)
# Backups directory
require.directory('/var/backups/postgresql', owner='postgres', use_sudo=True)
# Add crontab entry
fabtools.cron.add_task(config_name, '50 1 * * *', 'postgres', backup_script_path)
def install_nodejs():
"""
Test low level API
"""
from fabtools import nodejs
from fabtools import require
from fabtools.files import is_file
# Install Node.js from source
if nodejs.version() != nodejs.DEFAULT_VERSION:
nodejs.install_from_source()
assert is_file('/usr/local/bin/node')
assert nodejs.version() == nodejs.DEFAULT_VERSION
# Install / uninstall global package
if not nodejs.package_version('underscore'):
nodejs.install_package('underscore', version='1.4.2')
assert nodejs.package_version('underscore') == '1.4.2'
assert is_file('/usr/local/lib/node_modules/underscore/underscore.js')
nodejs.uninstall_package('underscore')
assert nodejs.package_version('underscore') is None
assert not is_file('/usr/local/lib/node_modules/underscore/underscore.js')
# Install / uninstall local package
if not nodejs.package_version('underscore', local=True):
nodejs.install_package('underscore', version='1.4.2', local=True)
assert is_file('node_modules/underscore/underscore.js')
assert nodejs.package_version('underscore', local=True) == '1.4.2'
nodejs.uninstall_package('underscore', local=True)
assert nodejs.package_version('underscore', local=True) is None
assert not is_file('node_modules/underscore/underscore.js')
# Install dependencies from package.json file
require.directory('nodetest')
with cd('nodetest'):
require.file('package.json',
contents=json.dumps({
'name': 'nodetest',
'version': '1.0.0',
'dependencies': {
'underscore': '1.4.2'
}
}))
nodejs.install_dependencies()
assert is_file('node_modules/underscore/underscore.js')
assert nodejs.package_version('underscore', local=True) == '1.4.2'
def update_config(config_path):
"""Update the Hadroid config."""
require.file(
path=env.hadroid_config,
source=config_path,
owner=env.app_user,
group=env.app_user,
mode='0600',
use_sudo=True,
)
def provision():
updateIndex()
require.file('.vimrc', source='vimrc')
# install java7
require.oracle_jdk.installed()
# install tomcat7
require.tomcat.installed()
# install other packages
require.deb.packages(['build-essential', 'python'], update=True)
deb.upgrade()
def test_default_temp_dir(self, is_file, md5sum, put, umask, owner, mode):
owner.return_value = 'root'
umask.return_value = '0002'
mode.return_value = '0664'
from fabtools import require
require.file('/var/tmp/foo', source=__file__, use_sudo=True)
put.assert_called_with(__file__,
'/var/tmp/foo',
use_sudo=True,
temp_dir='/tmp')
def setup_opsin():
"""Initial OPSIN setup."""
require.deb.packages(['openjdk-7-jre'])
with cd('/opt'):
require.file(url='https://bitbucket.org/dan2097/opsin/downloads/opsin-2.1.0-jar-with-dependencies.jar')
require.file(
path='/opt/opsin',
contents='#!/bin/bash\nexec java -jar /opt/opsin-2.1.0-jar-with-dependencies.jar "$@"\n',
mode='755'
)
def update_config(config_path):
"""Update the Hadroid config."""
require.file(
path=env.hadroid_config,
source=config_path,
owner=env.app_user,
group=env.app_user,
mode='0600',
use_sudo=True,
)
def provision():
updateIndex()
require.file(".vimrc", source="vimrc")
# install java7
require.oracle_jdk.installed()
# install tomcat7
require.tomcat.installed()
# install other packages
require.deb.packages(["build-essential", "python"], update=True)
deb.upgrade()
def install_nodejs():
"""
Test low level API
"""
from fabtools import nodejs
from fabtools import require
from fabtools.files import is_file
# Install Node.js from source
if nodejs.version() != nodejs.DEFAULT_VERSION:
nodejs.install_from_source()
assert is_file('/usr/local/bin/node')
assert nodejs.version() == nodejs.DEFAULT_VERSION
# Install / uninstall global package
if not nodejs.package_version('underscore'):
nodejs.install_package('underscore', version='1.4.2')
assert nodejs.package_version('underscore') == '1.4.2'
assert is_file('/usr/local/lib/node_modules/underscore/underscore.js')
nodejs.uninstall_package('underscore')
assert nodejs.package_version('underscore') is None
assert not is_file('/usr/local/lib/node_modules/underscore/underscore.js')
# Install / uninstall local package
if not nodejs.package_version('underscore', local=True):
nodejs.install_package('underscore', version='1.4.2', local=True)
assert is_file('node_modules/underscore/underscore.js')
assert nodejs.package_version('underscore', local=True) == '1.4.2'
nodejs.uninstall_package('underscore', local=True)
assert nodejs.package_version('underscore', local=True) is None
assert not is_file('node_modules/underscore/underscore.js')
# Install dependencies from package.json file
require.directory('nodetest')
with cd('nodetest'):
require.file('package.json', contents=json.dumps({
'name': 'nodetest',
'version': '1.0.0',
'dependencies': {
'underscore': '1.4.2'
}
}))
nodejs.install_dependencies()
assert is_file('node_modules/underscore/underscore.js')
assert nodejs.package_version('underscore', local=True) == '1.4.2'
def _update_varnish_sites(directory):
sites = run('find %s -mindepth 1 -maxdepth 1 -type f ' % directory,
combine_stderr=False).splitlines()
includes = ''.join('include "%s";\n' % s for s in sites)
# work around requrie.files(contents='') not replacing
contents = '# autogenerated\n%s' % includes
require.file('/etc/varnish/sites.vcl',
contents=contents,
use_sudo=True,
mode='644')
service.restart('varnish')
def pip_wraper():
with lcd('/home/ihor/.pip'):
require.directory('~/.pip', owner='web')
#require.file('~/.pip/pip.conf', source='pip.conf')
text = """
export WORKON_HOME=$HOME/.virtualenvs
export PROJECT_HOME=$HOME/Devel
source /usr/local/bin/virtualenvwrapper.sh
"""
require.directory('~/.virtualenvs', owner='web')
require.file('~/.bashrc', contents=text)
def refresh_uwsgi():
require.file("/etc/uwsgi/apps-available/%s.ini" % env.app_name,
contents=UWSGI_CONFIG_TPL % {
'user': env.user,
'group': env.group,
'app_root': env.app_root,
'app_env': env.app_env,
}, use_sudo=True)
sudo("ln -sf /etc/uwsgi/apps-available/%s.ini "
"/etc/uwsgi/apps-enabled/%s.ini" % (env.app_name, env.app_name))
require.service.restarted('uwsgi')
def default_locale(name):
"""
Require the locale to be the default.
"""
from fabtools import require
# Ensure the locale is available
locale(name)
# Make it the default
contents = 'LANG="%s"\n' % name
require.file('/etc/default/locale', contents, use_sudo=True)
def provision():
require.file('.vimrc', source='vimrc')
require.deb.packages([
'build-essential',
'openjdk-7-jdk',
'maven',
'ant',
'rake',
'git',
'vim'
], update=True)
deb.upgrade()
def default_locale(name):
"""
Require the locale to be the default
"""
from fabtools import require
# Ensure the locale is available
locale(name)
# Make it the default
contents = 'LANG="%s"\n' % name
require.file('/etc/default/locale', contents, use_sudo=True)
def refresh_uwsgi():
require.file("/etc/uwsgi/apps-available/%s.ini" % env.app_name,
contents=UWSGI_CONFIG_TPL % {
'user': env.user,
'group': env.group,
'app_root': env.app_root,
'app_env': env.app_env,
},
use_sudo=True)
sudo("ln -sf /etc/uwsgi/apps-available/%s.ini "
"/etc/uwsgi/apps-enabled/%s.ini" % (env.app_name, env.app_name))
require.service.restarted('uwsgi')
def qemu(central='127.0.0.1'):
require_man(7, 'online-testing', use_sudo=True)
if iputils.is_my_ip(central):
pkg('libvirt0 qemu-kvm')
require.file('/usr/local/bin/emu-vnc',
source='files/bin/emu-vnc', owner='root', group='root',
mode='755',
verify_remote=True, use_sudo=True)
else:
pkg('nfs-common')
mount_line('%s:/var/lib/libvirt/images /var/www nfs defaults 0 0' %
central)
def setup_certbot():
require.file('/var/do.ini', source='../__KEYS__/DO_staging.ini')
require.deb.uptodate_index()
require.deb.packages(
['software-properties-common', 'python3-certbot-dns-digitalocean'])
sudo('add-apt-repository universe')
sudo('add-apt-repository ppa:certbot/certbot')
require.deb.uptodate_index()
sudo('apt-get install certbot python-certbot-nginx')
sudo('certbot certonly -a dns-digitalocean -i nginx'
' -d "*.reckonsys.xyz" -d reckonsys.xyz'
' --server https://acme-v02.api.letsencrypt.org/directory')
def require_config(filepath, app, ctx):
# We only set add a setting clld.files, if the corresponding directory exists;
# otherwise the app would throw an error on startup.
files_dir = app.www_dir / 'files'
files = files_dir if exists(str(files_dir)) else None
sudo_upload_template('config.ini', dest=str(filepath), context=ctx, files=files)
if app.stack == 'django' and confirm('Recreate secret key?', default=True):
key_chars = "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)"
secret_key = "".join([random.choice(key_chars) for i in range(50)])
require.file(
str(filepath.parent / 'secret_key'), contents=secret_key, use_sudo=True, mode='644')
def deploy_celery():
"""Deploy celery service scripts to appropriate location."""
require.file('/etc/init.d/%(app_name)s-celeryd' % env, source='deploy/celeryd' % env, use_sudo=True, mode='755')
require.files.template_file(
'/etc/default/%(app_name)s-celeryd' % env,
template_source='deploy/celeryd-default' % env,
context=dict(app_name=env.app_name, app_dir=env.app_dir),
use_sudo=True
)
sudo('update-rc.d %(app_name)s-celeryd defaults' % env)
require.directory('/var/log/%(app_name)s' % env, use_sudo=True, owner=env.app_user, group='www-data')
require.directory('/var/run/%(app_name)s' % env, use_sudo=True, owner=env.app_user, group='www-data')
def nominatim():
pgconfig(for_import=True)
with cd('/opt/osm'):
nominatime_archive = 'Nominatim-%s.tar.bz2' % config.NOMINATIM_VERSION
nominatim_url = 'http://www.nominatim.org/release/'\
+ nominatime_archive
require.file(url=nominatim_url, use_sudo=True, owner=config.GIS_USER)
nominatim_dir = sudo('''tar tf %s | sed -e 's@/.*@@' | uniq''' %
nominatime_archive,
user=config.GIS_USER)
sudo('''tar xvf ''' + nominatime_archive, user=config.GIS_USER)
with cd(nominatim_dir):
sudo('./autogen.sh', user=config.GIS_USER)
sudo('./configure', user=config.GIS_USER)
sudo('make', user=config.GIS_USER)
context = {
'db_name': config.GIS_DB,
'db_user': config.GIS_USER,
'db_passowrd': config.GIS_PASSWORD,
}
require.files.template_file(path='settings/local.php',
template_source='templates/local.php',
context=context,
use_sudo=True,
owner=config.GIS_USER)
with cd('data'):
wiki_urls = [
'http://www.nominatim.org/data/wikipedia_article.sql.bin',
'http://www.nominatim.org/data/wikipedia_redirect.sql.bin'
]
for url in wiki_urls:
require.file(url=url, use_sudo=True, owner=config.GIS_USER)
sudo('./utils/setup.php --osm-file %s --all --osm2pgsql-cache %d' %
(pbf_path(), config.RAM_SIZE / 4 * 3),
user=config.GIS_USER)
sudo('./utils/specialphrases.php --countries > sp_countries.sql',
user=config.GIS_USER)
sudo('psql -d %s -f sp_countries.sql' % config.GIS_DB,
user=config.GIS_USER)
sudo('./utils/specialphrases.php --wiki-import > sp.sql',
user=config.GIS_USER)
sudo('psql -d %s -f sp.sql' % config.GIS_DB, user=config.GIS_USER)
require.directory('/var/www/nominatim',
mode='755',
owner=config.GIS_USER,
use_sudo=True)
sudo('./utils/setup.php --create-website /var/www/nominatim',
user=config.GIS_USER)
require.apache.site('200-nominatim.conf',
template_source='templates/200-nominatim.conf')
require.service.restarted('apache2')
pgconfig(for_import=False)
def setup():
local_clone_repos()
require.files.directories([apps_dir, logs_dir])
require.files.directories(["/var/repo/"], use_sudo=True)
require.deb.package("software-properties-common")
sudo("add-apt-repository ppa:ondrej/php")
sudo("add-apt-repository ppa:ondrej/apache2")
sudo("add-apt-repository ppa:certbot/certbot")
deb.update_index()
deb.upgrade()
require.git.command()
require.deb.packages([
"mysql-server", "dpkg-dev", "php7.1", "php7.1-mysql", "php7.1-gd",
"php7.1-dev", "php7.1-curl", "php7.1-cli", "php7.1-json", "php-apcu",
"libpcre3-dev", "php-pear", "libapache2-mod-php7.1", "sendmail",
"php7.1-mbstring", "python-pip", "python-certbot-apache"
])
sudo("yes '' | pecl install apc")
sudo("pip install pygments")
require.apache.server()
for apache_mod in apache_mods:
require.apache.module_enabled(apache_mod)
push_repos()
require.apache.site(
hostname,
template_contents=CONFIG_TPL,
port=80,
hostname=hostname,
docroot=docroot,
)
require.apache.site_enabled(hostname)
require.apache.site_disabled('default')
print("Please make sure %s is pointing to %s before certbot install" %
(hostname, env.hosts))
sudo("certbot --apache")
require.mysql.server(password=db_root_pass)
with settings(mysql_user='******', mysql_password=db_root_pass):
require.mysql.user(env.user, db_user_pass)
grant_all(env.user)
require.file("/etc/mysql/conf.d/mysql.cnf",
"[mysqld]\nsql_mode=STRICT_ALL_TABLES",
use_sudo=True)
sudo("service mysql restart")
require.nodejs.installed_from_source(version='8.9.1')
with cd("%s/phabricator" % apps_dir):
run("./bin/config set mysql.host localhost")
run("./bin/config set mysql.user %s" % env.user)
run("./bin/config set mysql.pass %s" % db_user_pass)
run("./bin/config set phabricator.base-uri 'https://%s'" % hostname)
run("./bin/storage upgrade --force")
run("./bin/phd start")
run("./bin/aphlict start")
def setup_rdkit():
"""Initial RDKit setup."""
require.deb.packages([
'build-essential', 'python-numpy', 'cmake', 'python-dev', 'sqlite3',
'libsqlite3-dev', 'libboost-dev', 'libboost-system-dev',
'libboost-thread-dev', 'libboost-serialization-dev',
'libboost-python-dev', 'libboost-regex-dev', 'wget'
])
with cd('/opt'):
require.file(
url=
'https://github.com/rdkit/rdkit/archive/Release_2016_03_1.tar.gz')
sudo('tar -xvf Release_2016_03_1.tar.gz')
def setup_project():
"""
Require working copy of project cloned from remote repository.
"""
require.directory(env.project_dir,
use_sudo=True,
owner=env.user,
group="www-data")
with cd(env.project_dir):
require.git.working_copy(env.config["repository"], directly=True)
require.file(
os.path.join(env.project_dir, env.config["project"],
"secrets.json"), json.dumps(env.secrets))
def copy_downloads(app, source_dir, pattern='*'):
"""copy downloads for the app"""
require.directory(str(app.download_dir), use_sudo=True, mode='777')
source_dir = pathlib.Path(source_dir)
for f in source_dir.glob(pattern):
require.file(str(app.download_dir / f.name),
source=f,
use_sudo=True,
owner=app.name,
group=app.name)
require.directory(str(app.download_dir), use_sudo=True, mode='755')
def checkout(url, workdir, name):
if not exists(env.deploy_key_path):
run('mkdir -p %s/.ssh' % env.home)
require.file(env.home + '/.ssh/config', source='_ops/ssh_config')
require.file(env.deploy_key_path, source='/Users/alankang/.ssh/id_rsa')
run('chmod 0600 ' + env.deploy_key_path)
if exists(os.path.join(workdir, name)):
with cd(os.path.join(workdir, name)):
run("ssh-agent bash -c 'ssh-add %s; git pull'" % env.deploy_key_path)
else:
with cd(workdir):
run("ssh-agent bash -c 'ssh-add %s; git clone %s %s'" % (env.deploy_key_path, url, name))
