def delete():
if not confirm("Are you sure you want to delete the bucket %r?"
% bucket_name):
abort('Aborting at user request.')
conn = connect_s3()
conn.delete_bucket(bucket_name)
print 'Bucket %r deleted.' % bucket_name
def deploy_code():
def check_if_code_exists():
""" Check if a branch or tag exists in the user's local repo. """
# Check if the branch exists
with settings(warn_only=True):
if local('git show-ref --heads --tags %s' % env.code).failed:
return False
else:
return True
def get_checked_out_branch_or_tag():
return run('git name-rev --name-only HEAD')
# Deploy the code
with cd(env.code_dir):
if not check_if_code_exists():
abort('The "%s" %s does not exist in your repo!'
% (env.code, env.code_type))
if not confirm('Current checked out %s is "%s", proceed with \
deploying "%s"?' % (env.code_type, get_checked_out_branch_or_tag(), env.code)):
abort('Canceled deployment')
run('git fetch')
run('git fetch --tags')
run('git checkout %s' % env.code)
if env.code_type == 'branch':
run('git pull origin %s' % env.code)
def dump_database(database, path='/var/backups/postgres', filename='', format='plain', port=None):
"""
Generate a dump database to a remote destination path
Example::
import fabtools
fabtools.postgres.dump_database('myapp', path='/var/backups/postgres', filename='myapp-backup.sql')
# If not filename specified will be saved with the date file format: database-201312010000.sql
fabtools.postgres.dump_database('myapp', path='/var/backups/postgres')
# If not path specified will be saved at '/var/backups/postgres'
fabtools.postgres.dump_database('myapp')
# You can scpecify the pg_dump's custom format (able to restore with pg_restore)
fabtools.postgres.dump_database('myapp', format='custom')
"""
command_options = []
port_option = _port_option(port)
if port_option is not None:
command_options.append(port_option)
command_options = ' '.join(command_options)
if fabtools.files.is_dir(path):
if database_exists(database):
date = _date.today().strftime("%Y%m%d%H%M")
if not filename:
filename = '%(database)s-%(date)s.sql' % locals()
dest = quote(posixpath.join(path, filename))
_run_as_pg('pg_dump %(command_options)s %(database)s --format=%(format)s --blobs --file=%(dest)s' % locals())
else:
abort('''Database does not exist: %(database)s''' % locals())
else:
abort('''Destination path does not exist: %(path)s''' % locals())
def validate_branch(branch):
if branch == "override":
pass
elif branch is None:
abort("must specify a branch")
elif branch not in PUBLISHED_BRANCHES:
abort("must specify a published branch.")
def release():
"""Performs a full release"""
with cd(getcwd()):
with msg("Creating env"):
run("mkvirtualenv test")
with msg("Building"):
with prefix("workon test"):
run("fab develop")
with msg("Running tests"):
with prefix("workon test"):
run("fab test")
with msg("Building docs"):
with prefix("workon test"):
run("pip install -r docs/requirements.txt")
run("fab docs")
version = run("python setup.py --version")
if "dev" in version:
abort("Detected Development Version!")
print("Release version: {0:s}".format(version))
if prompt("Is this ok?", default="Y", validate=r"^[YyNn]?$") in "yY":
run("git tag {0:s}".format(version))
run("python setup.py egg_info sdist bdist_egg bdist_wheel register upload")
run("python setup.py build_sphinx upload_sphinx")
with msg("Destroying env"):
run("rmvirtualenv test")
def build(treeish='head'):
"""Build a release."""
version = local("git describe {}".format(treeish), capture=True)
with settings(hide('warnings'), warn_only=True):
cmd = "git diff-index --quiet {} --".format(treeish)
is_committed = local(cmd).succeeded
cmd = "git branch -r --contains {}".format(version)
is_pushed = local(cmd, capture=True)
if not is_committed:
prompt = "Uncommitted changes. Continue?"
if not confirm(prompt, default=False):
abort("Canceled.")
if not is_pushed:
prompt = "Commit not pushed. Continue?"
if not confirm(question=prompt, default=False):
abort("Canceled.")
output = "/tmp/{}.tar.gz".format(version)
prefix = "{}/".format(version)
cmd = "git archive --prefix={prefix} --format=tar.gz --output={output} {version}:src"
local(cmd.format(prefix=prefix, output=output, version=version))
puts("\nBuilt: {} at: {}".format(version, output))
return output
def _standby_clone():
""" With "node1" server running, we want to use the clone standby
command in repmgr to copy over the entire PostgreSQL database cluster
onto the "node2" server. """
# manualy:
# $ mkdir -p /var/lib/postgresql/9.1/testscluster/
# $ rsync -avz --rsh='ssh -p2222' [email protected]:/var/lib/postgresql/9.1/testscluster/ /var/lib/postgresql/9.1/testscluster/
with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
puts(green('Start cloning the master'))
repmgr_clone_command = 'repmgr -D %(slave_pgdata_path)s -d %(sync_db)s -p %(cluster_port)s -U %(sync_user)s -R postgres --verbose standby clone %(pgmaster_ip)s' % env
puts(green(repmgr_clone_command))
puts("-" * 40)
res = sudo(repmgr_clone_command, user='******')
if 'Can not connect to the remote host' in res or 'Connection to database failed' in res:
puts("-" * 40)
puts(green(repmgr_clone_command))
puts("-" * 40)
puts("Master server is %s reachable." % red("NOT"))
puts("%s you can try to CLONE the slave manually [%s]:" % (green("BUT"), red("at your own risk")))
puts("On the slave server:")
puts("$ sudo -u postgres rsync -avz --rsh='ssh -p%(master_ssh_port)s' postgres@%(pgmaster_ip)s:%(master_pgdata_path)s %(slave_pgdata_path)s --exclude=pg_xlog* --exclude=pg_control --exclude=*.pid" % env)
puts("Here:")
puts("$ fab <cluster_task_name> finish_configuring_slave")
abort("STOP...")
def upload(tag=False):
"""Upload new zip files.
@param tag Use git tag instead of hash?
"""
suffix = get_suffix(tag)
base_zip = "bootstrap.zip"
suffix_zip = "bootstrap-%s.zip" % suffix
if not (os.path.exists(base_zip) and os.path.exists(suffix_zip)):
abort("Did not find current zip files. Please create.")
# Check if existing downloads
github = GitHub()
dl_dict = dict((x['name'], x) for x in github.downloads())
dl_suffix = dl_dict.get(suffix_zip)
dl_base = dl_dict.get(base_zip)
if dl_suffix is not None:
print("Found suffixed zip file already. Skipping")
return
if dl_base is not None:
print("Removing current base zip file.")
result = github.downloads_del(dl_base)
print("Result: %s" % json.dumps(result, indent=2))
print("Upload new base zip file.")
result = github.downloads_put(base_zip, suffix)
print("\nResult: %s" % json.dumps(result, indent=2))
print("Upload new suffixed zip file.")
result = github.downloads_put(suffix_zip, suffix)
print("\nResult: %s" % json.dumps(result, indent=2))
def vcs_upload():
"""
Uploads the project with the selected VCS tool.
"""
if env.deploy_tool == "git":
remote_path = "ssh://%s@%s%s" % (env.user, env.host_string,
env.repo_path)
if not exists(env.repo_path):
run("mkdir -p %s" % env.repo_path)
with cd(env.repo_path):
run("git init --bare")
local("git push -f %s master" % remote_path)
with cd(env.repo_path):
run("GIT_WORK_TREE=%s git checkout -f master" % env.proj_path)
run("GIT_WORK_TREE=%s git reset --hard" % env.proj_path)
elif env.deploy_tool == "hg":
remote_path = "ssh://%s@%s/%s" % (env.user, env.host_string,
env.repo_path)
with cd(env.repo_path):
if not exists("%s/.hg" % env.repo_path):
run("hg init")
print(env.repo_path)
with fab_settings(warn_only=True):
push = local("hg push -f %s" % remote_path)
if push.return_code == 255:
abort()
run("hg update")
def safe_reboot():
"""Reboot a mongo machine, stepping down if it is the primary"""
import vm
if not vm.reboot_required():
print("No reboot required")
return
while True:
if cluster_is_ok():
break
sleep(5)
print("Waiting for cluster to be okay")
primary = _find_primary()
if primary == 'No primary currently elected':
return primary
if i_am_primary(primary):
execute(step_down_primary)
for i in range(5):
if cluster_is_ok() and not i_am_primary():
break
sleep(1)
if not cluster_is_ok() or i_am_primary():
abort("Cluster has not recovered")
execute(vm.reboot, hosts=[env['host_string']])
def deploy():
print green("Begining update...")
print ""
print blue('Checking pre-requisites...')
print cyan("Checking for local changes...")
has_changes = local("git status --porcelain", capture=True)
if REQUIRE_CLEAN and has_changes:
abort(red("Your working directory is not clean."))
print cyan("Ensuring remote working area is clean...")
GIT_CMD = "git --work-tree={0} --git-dir={0}/.git".format(DEPLOY_PATH)
has_changes = run(GIT_CMD + " status --porcelain")
if has_changes:
abort(red("Remote working directory is not clean."))
print blue("Finished checking pre-requisites.")
print ""
print green("Starting deployment...")
print ""
print green("Updating environment...")
with cd(DEPLOY_PATH):
print cyan("Pulling from master")
run('git pull')
sudo('reboot')
def ensure_running(retries=15, wait=10):
"""Ensure cassandra is running on all nodes.
Runs 'nodetool ring' on a single node continuously until it
reaches the specified number of retries.
INTENDED TO BE RUN ON ONE NODE, NOT ALL.
"""
time.sleep(15)
for attempt in range(retries):
ring = StringIO(fab.run('JAVA_HOME={java_home} {nodetool_bin} ring'.format(
java_home=config['java_home'], nodetool_bin=_nodetool_cmd())))
broadcast_ips = [x.get('external_ip', x['internal_ip']) for x in config['hosts'].values()]
nodes_up = dict((host,False) for host in broadcast_ips)
for line in ring:
for host in broadcast_ips:
try:
if host in line and " Up " in line:
nodes_up[host] = True
except UnicodeDecodeError:
# sometimes the operations on line will fail when it contains characters
# outside ascii range. we don't care; these lines can exist as long as
# we see the 'UP' lines that let us know a node is up.
pass
for node,up in nodes_up.items():
if not up:
fab.puts("Node is not up (yet): %s" % node)
if False not in nodes_up.values():
fab.puts("All nodes available!")
return
fab.puts("waiting %d seconds to try again.." % wait)
time.sleep(wait)
else:
fab.abort("Timed out waiting for all nodes to startup")
def database(name, owner, template='template0', encoding='UTF8',
locale='en_US.UTF-8', allow_restart=False):
"""
Require a PostgreSQL database.
::
from fabtools import require
require.postgres.database('myapp', owner='dbuser')
"""
locale_transform = lambda l: l.strip().lower().replace('-', '')
if not database_exists(name):
locales = map(
locale_transform,
run('locale -a').split()
)
if locale_transform(locale) not in locales:
if not allow_restart:
abort(
'New locale "{}" must be installed and '
'postgres must be restarted after that'.format(
locale
)
)
require_locale(locale)
restarted(_service_name())
create_database(name, owner, template=template, encoding=encoding,
locale=locale)
def update():
# test configuration start
if not test_configuration():
if not console.confirm("Configuration test %s! Do you want to continue?" % red_bg('failed'), default=False):
abort("Aborting at user request.")
# test configuration end
_verify_sudo()
if env.ask_confirmation:
if not console.confirm("Are you sure you want to deploy in %s?" % red_bg(env.project.upper()), default=False):
abort("Aborting at user request.")
puts(green_bg('Start deploy...'))
start_time = datetime.now()
if not 'synced_projectdir' in env or not env.synced_projectdir:
git_pull()
# _install_requirements()
# _upload_nginx_conf()
# _upload_rungunicorn_script()
# _upload_supervisord_conf()
_prepare_django_project()
_prepare_media_path()
_supervisor_restart()
end_time = datetime.now()
finish_message = '[%s] Correctly deployed in %i seconds' % \
(green_bg(end_time.strftime('%H:%M:%S')), (end_time - start_time).seconds)
puts(finish_message)
def staging():
answer = prompt('Are you sure you want to DELETE ALL DATA on "{0}" and replace it with test data? (type "I am sure" to continue):'.format(env.host_string))
if answer != 'I am sure':
abort('Aborted!')
run_fixtures('all')
def create_ami(instance_id, name):
"""
Create AMI image from specified instance
The instance needs to be shutdown before the creation begin.
"""
image_name = "{0}_{1}".format(name, datetime.now().strftime("%Y%m%d-%H%M"))
conn = boto.connect_ec2()
image_id = conn.create_image(instance_id=instance_id, name=image_name)
puts("Creating AMI {0} for instance {1}".format(image_name, image_id))
while True:
puts('.', end='')
sys.stdout.flush()
image = conn.get_image(image_id)
if image.state == 'available':
break
if image.state == "failed":
abort("Error creating AMI for {0}".format(image_id))
time.sleep(5.0)
puts("\nImage {0} created".format(image_name))
return image_id
def git_seed(repo_path, commit=None, ignore_untracked_files=False):
'''seed a remote git repository'''
commit = _get_commit(commit)
force = ('gitric_force_push' in env) and '-f' or ''
dirty_working_copy = _is_dirty(commit, ignore_untracked_files)
if dirty_working_copy and 'gitric_allow_dirty' not in env:
abort(
'Working copy is dirty. This check can be overridden by\n'
'importing gitric.api.allow_dirty and adding allow_dirty to your '
'call.')
# initialize the remote repository (idempotent)
run('git init %s' % repo_path)
# silence git complaints about pushes coming in on the current branch
# the pushes only seed the immutable object store and do not modify the
# working copy
run('GIT_DIR=%s/.git git config receive.denyCurrentBranch ignore' %
repo_path)
# a target doesn't need to keep track of which branch it is on so we always
# push to its "master"
with settings(warn_only=True):
push = local(
'git push git+ssh://%s@%s:%s%s %s:refs/heads/master %s' % (
env.user, env.host, env.port, repo_path, commit, force))
if push.failed:
abort(
'%s is a non-fast-forward\n'
'push. The seed will abort so you don\'t lose information. '
'If you are doing this\nintentionally import '
'gitric.api.force_push and add it to your call.' % commit)
def cmd(stackname, command=None, username=DEPLOY_USER, clean_output=False, concurrency=None, node=None):
if command is None:
abort("Please specify a command e.g. ./bldr cmd:%s,ls" % stackname)
LOG.info("Connecting to: %s", stackname)
instances = _check_want_to_be_running(stackname)
if not instances:
return
# take out the load of crap that Fabric prints mangling the useful output
# of a remote command
custom_settings = {}
if clean_output:
fabric.state.output['status'] = False
fabric.state.output['running'] = False
custom_settings['output_prefix'] = False
try:
with settings(**custom_settings):
return stack_all_ec2_nodes(
stackname,
(run, {'command': command}),
username=username,
abort_on_prompts=True,
concurrency=concurrency_for(stackname, concurrency),
node=int(node) if node else None
)
except FabricException as e:
LOG.error(e)
exit(2)
def run(self, username=None, pubkey=None, as_root=False):
if as_root:
remote_user = '******'
execute = run
else:
remote_user = env.local_user
execute = sudo
with settings(user=remote_user):
keyfile = Path(pubkey or Path('~', '.ssh', 'id_rsa.pub')).expand()
if not keyfile.exists():
abort('Public key file does not exist: %s' % keyfile)
with open(keyfile, 'r') as f:
pubkey = f.read(65535)
username = username or prompt('Username: '******'s password: "******"%s\", \"password\")\'' % (password),
capture=True)
for command in self.commands:
execute(command.format(**locals()))
def deploy(justPull=False):
"""
Update the remote instance.
Pull from git, update virtualenv, create static and restart gunicorn
"""
is_this_initial = False
if run("test -d %s/.git" % env.REPOSITORY_FOLDER,
quiet=True).failed: # destination folder to be created
message = 'Repository folder doesn\'t exists on destination. Proceed with initial deploy?'
if not confirm(message):
abort("Aborting at user request.")
else:
initial_deploy()
is_this_initial = True
for secret in secrets_file_paths():
if run("test -e %s" % posixpath.join(env.REPOSITORY_FOLDER, secret),
quiet=True).failed: # secrets missing
message = 'Some secret doesn\'t exists on destination. Proceed with initial deploy?'
send_secrets(ask=True)
update_instance(do_update_requirements=is_this_initial or DO_REQUIREMENTS, justPull=justPull)
restart()
def validate_branch(branch):
if branch == 'override':
pass
elif branch is None:
abort('must specify a branch')
elif branch not in PUBLISHED_BRANCHES:
abort('must specify a published branch.')
def install():
arch = info.arch()
if re.match('i?86', arch):
arch = 'x86'
pypyURL = pypyURLs.get(arch)
pypyDir = pypyDirs.get(arch)
if pypyURL is None or pypyDir is None:
abort("Can't install pypy on unknown architecture.")
sudo('/bin/mkdir -p /opt')
if fails('/usr/bin/id {}'.format('pypy')):
sudo('/usr/sbin/useradd --home-dir {} --gid bin '
'-M --system --shell /bin/false '
'pypy'.format(pypyDir))
else:
sudo('/usr/sbin/usermod --home {} pypy'.format(pypyDir))
with cd('/opt'):
for url in pypyURL, pipURL:
sudo('/usr/bin/wget -nc {}'.format(url))
sudo('/bin/tar xf {}'.format(path.basename(pypyURL)))
sudo('~pypy/bin/pypy {}'.format(path.join('/opt/', path.basename(pipURL))), pty=False)
sudo('~pypy/bin/pip install pyopenssl')
sudo('~pypy/bin/pip install svn+svn://svn.twistedmatrix.com/svn/Twisted/trunk/')
def setup():
# test configuration start
if not test_configuration():
if not console.confirm("Configuration test %s! Do you want to continue?" % red_bg('failed'), default=False):
abort("Aborting at user request.")
# test configuration end
if env.ask_confirmation:
if not console.confirm("Are you sure you want to setup %s?" % red_bg(env.project.upper()), default=False):
abort("Aborting at user request.")
puts(green_bg('Start setup...'))
start_time = datetime.now()
_verify_sudo
_install_dependencies()
_create_django_user()
_setup_directories()
_git_clone()
_install_virtualenv()
_create_virtualenv()
_install_gunicorn()
_install_requirements()
_upload_nginx_conf()
_upload_rungunicorn_script()
_upload_supervisord_conf()
end_time = datetime.now()
finish_message = '[%s] Correctly finished in %i seconds' % \
(green_bg(end_time.strftime('%H:%M:%S')), (end_time - start_time).seconds)
puts(finish_message)
def json_output():
if env.input_file is None or env.output_file is None:
abort('[json]: you must specify input and output files.')
with open(env.input_file, 'r') as f:
document = f.read()
doc = json.loads(document)
if 'body' not in doc:
pass
else:
text = doc['body'].encode('ascii', 'ignore')
text = re.sub('<[^>]*>', '', text)
text = re.sub('“', '"', text)
text = re.sub('”', '"', text)
text = re.sub('‘', "'", text)
text = re.sub('’', "'", text)
text = re.sub('&#\d{4};', '', text)
doc['text'] = ' '.join(text.split('\n')).strip()
url = [ 'http://docs.mongodb.org', get_manual_path() ]
url.extend(env.input_file.rsplit('.', 1)[0].split('/')[3:])
doc['url'] = '/'.join(url)
with open(env.output_file, 'w') as f:
f.write(json.dumps(doc))
def git_clone(repo):
topsrcdir = repo_check(repo, check_path=False, workdir=False)
try:
repo_url = dict(env.CFG_INVENIO_REPOS)[repo]['repository']
except KeyError:
abort(red("Repository URL for %s not defined" % repo))
basename = os.path.basename(topsrcdir)
parent = os.path.dirname(topsrcdir)
if os.path.exists(topsrcdir):
res = confirm("Remove existing source code in %s ?" % topsrcdir)
if not res:
abort(red("Cannot continue") % env)
else:
local("rm -Rf %s" % topsrcdir)
else:
if not os.path.exists(parent):
local("mkdir -p %s" % parent)
ctx = {
'basename': basename,
'parent': parent,
'topsrcdir': topsrcdir,
'url': repo_url,
}
local("cd %(parent)s; git clone %(url)s %(basename)s " % ctx)
def repo_setup(repo, ref):
""" Clone repository """
puts(cyan(">>> Setting up repository %s with ref %s..." % (repo, ref)))
topsrcdir = repo_check(repo, check_path=False, workdir=False)
workdir = repo_check(repo, check_path=False, workdir=True)
gitdir = os.path.join(topsrcdir, '.git')
if not os.path.exists(env.CFG_SRCDIR):
res = confirm("Create repository root %s?" % env.CFG_SRCDIR)
if not res:
abort(red("Cannot continue") % env)
else:
local("mkdir -p %s" % env.CFG_SRCDIR)
if not os.path.exists(gitdir) and os.path.exists(topsrcdir):
res = confirm("Remove %s (it does not seem to be a git repository)?" % topsrcdir)
if not res:
abort(red("Cannot continue") % env)
else:
local("rm -Rf %s" % topsrcdir)
if not os.path.exists(gitdir):
git_clone(repo)
if not os.path.exists(workdir):
git_newworkdir(repo)
git_checkout(repo, ref)
repo_prepare(repo)
def config(self):
self.nodetype = utils.to_list(self.nodetype)
self.siteinfo = utils.to_list(self.siteinfo)
if not self.nodetype or not self.siteinfo:
raise exception.ConfigException(("Could not run YAIM: Bad "
"nodetype or site-info."))
with tempfile.NamedTemporaryFile("w+t",
dir=config.CFG["yaim_path"],
delete=True) as f:
for si in self.siteinfo:
f.write("source %s\n" % si)
f.flush()
api.info(("Creating temporary file '%s' with "
"content: %s" % (f.name, f.readlines())))
# NOTE(orviz) Cannot use 'capture=True': execution gets
# stalled (defunct)
with context_managers.lcd(config.CFG["yaim_path"]):
abort_exception_default = fabric_api.env.abort_exception
fabric_api.env.abort_exception = exception.ConfigException
try:
fabric_api.local("/opt/glite/yaim/bin/yaim -c -s %s -n %s"
% (f.name, " -n ".join(self.nodetype)))
except exception.ConfigException:
fabric_api.abort(api.fail(("YAIM execution failed. Check "
"the logs at '/opt/glite/yaim/"
"log/yaimlog'.")))
api.info("YAIM configuration ran successfully.")
fabric_api.env.abort_exception = abort_exception_default
def _test_install(folder, box):
"""
Test installation of cozy
Start vagrant box
Install cozy on this box
Test if cozy is well installed
"""
if not folder or not box:
abort('_test_install : both parameters are needed')
if not is_dir(folder):
local('mkdir ' + folder)
with lcd(folder):
start_box(box)
try:
install_cozy()
_test_status(5)
_test_register()
_test_bad_register()
_test_install_app()
_test_uninstall_app()
except Exception, e:
local('vagrant halt -f')
traceback.print_exc()
sys.exit(1)
else:
def diff(self, old, new, rev1, rev2, user, passwd):
if old == new and rev1 == rev2:
abort('can not deploy a duplicated (version,revision):(%s,%s)' \
% (new.tag, rev1))
if user != None and passwd != None:
cmd = ['svn --username=%s --password=%s --non-interactive --trust-server-cert diff --summarize' % (user, passwd),
'--old=' + str(old),
'--new=' + str(new),
]
else:
cmd = ['svn --non-interactive --trust-server-cert diff --summarize',
'--old=' + str(old),
'--new=' + str(new),
]
if rev1 and rev2:
cmd.append('-r%s:%s' % (rev1, rev2))
svnlog = local(' '.join(cmd), capture=True).stdout.strip()
if not svnlog:
abort('no diff between two (version,revision):(%s,%s)(%s,%s)' \
% (old.tag, new.tag, rev1, rev2))
print 'CHANGE HISTORY:'
print svnlog
print '-'*10
return svnlog
def deploy_static():
if not os.path.exists(".build") and not confirm("Looks like there is no build. Continue anyway? [DANGEROUS]"):
abort("Aborting at user request. Type \"fab build\" before deployment.")
require('hosts', provided_by=[staging, prod])
require('staticpath', provided_by=[staging,prod])
_put_dir('static', env.staticpath )
import os
import imp
import datetime
from fabric.api import execute, task, env, abort, local
# this will print paramiko errors on stderr
import logging
logging.getLogger('paramiko.transport').addHandler(logging.StreamHandler())
# if the env variable teacup_config is set import the config file specified.
# the variable is set as follows: fab --set teacup_config=myconfig.py
# otherwise import the default config.py
try:
if not os.path.isfile(env.teacup_config):
abort('Specified config file %s does not exit' % env.teacup_config)
# before loading config change to the directory of specified config file
# this is needed to make execfile without absolute path in config file work
# afterwards change back directory to current path
curr_dir = os.getcwd()
config_dir = os.path.dirname(env.teacup_config)
if config_dir == '':
config_dir = '.'
os.chdir(config_dir)
config = imp.load_source('config', env.teacup_config)
os.chdir(curr_dir)
except AttributeError:
import config
def deploy():
def docker_compose(command):
with cd(PATH):
with shell_env(CI_BUILD_REF_NAME=os.getenv(
'CI_BUILD_REF_NAME', 'master')):
run('set -o pipefail; docker-compose %s | tee' % command)
variables_set = True
for var in VARIABLES + ('CI_BUILD_TOKEN',):
if os.getenv(var) is None:
variables_set = False
print(red('ERROR: environment variable ' + var + ' is not set.'))
if not variables_set:
abort('Missing required parameters')
with hide('commands'):
run('rm -f "%s"' % ENV_FILE)
append(ENV_FILE,
['%s=%s' % (var, val) for var, val in zip(
VARIABLES, map(os.getenv, VARIABLES))])
run('cat /srv/gitlab-login | docker login --username %s %s --password-stdin ' % (
os.getenv('REGISTRY_USER', 'gitlab-ci-token'),
os.getenv('CI_REGISTRY','registry.gitlab.com')))
# run('docker login -u %s -p %s %s' % (os.getenv('REGISTRY_USER',
# 'gitlab-ci-token'),
# os.getenv('CI_BUILD_TOKEN'),
# os.getenv('CI_REGISTRY',
# 'registry.gitlab.com')))
put('docker-compose.prod.yml', PATH)
put('app.sh', PATH)
run('cp ' + PATH + '/docker-compose.prod.yml ' + PATH + '/docker-compose.yml')
run('rm -f ' + PATH + '/docker-compose.prod.yml')
run('source %s' % ENV_FILE)
docker_compose("pull")
# run('docker stop $(docker ps -a -q)')
# try:
# run("docker network disconnect pye_default pye_web_1 -f")
# except:
# pass
#
# try:
# run("docker network disconnect pye_default $(docker ps -a --filter=\"name=_pye_web\" --format '{{.Names}}') -f")
# except:
# pass
# remove static volume
docker_compose('-p boilerplate down')
run('docker volume rm -f boilerplate_static_files')
docker_compose('-p boilerplate up -d')
def run_command_to_web(command):
run("docker exec -it boilerplate_web_1 " + command)
# try:
# run("docker exec -it $(docker ps -a --filter=\"name=_pye_web\" --format '{{.Names}}') " + command)
# except:
# pass
# try:
# run("docker exec -it pye_web_1" + command)
# except:
# pass
with cd(PATH):
try:
run('./app.sh cert')
except:
pass
run('./app.sh migrate')
run('./app.sh collectstatic')
# run_command_to_web('python manage.py migrate')
# run_command_to_web('bash -c "cd app && npm i && npm run build"')
# run_command_to_web('python manage.py collectstatic --noinput')
# ssh letsencrypt
run('docker kill -s HUP boilerplate_nginx_1')
def test():
with settings(warn_only=True):
result = local("nosetests -v", capture=True)
if result.failed and not confirm("Tests failed. Continue?"):
abort("Aborted at user request.")
def install():
current_dir = os.path.abspath(os.curdir)
# check hardware support
dist = check_system_support()
# install dependent packages
with settings(hide('stdout'), warn_only=True):
cmd = "sudo apt-get update"
sudo(cmd)
with settings(hide('running'), warn_only=True):
cmd = "apt-get install --force-yes -y qemu-kvm libvirt-bin libglu1-mesa "
cmd += "gvncviewer python-dev python-libvirt python-lxml python-lzma "
cmd += "apparmor-utils libc6-i386 python-pip libxml2-dev libxslt1-dev apache2"
if dist == "precise":
cmd += " python-xdelta3"
if sudo(cmd).failed:
abort("Failed to install libraries")
else:
if sudo(cmd).failed:
abort("Failed to install libraries")
# Python-xdelta3 is no longer supported in Ubuntu 14.04 LTS.
# But you can install deb of Ubunutu 12.04 at Ubuntu 14.04.
with cd(current_dir):
package_name = "python-xdelta3.deb"
cmd = "wget http://mirrors.kernel.org/ubuntu/pool/universe/x/xdelta3/python-xdelta3_3.0.0.dfsg-1build1_amd64.deb -O %s" % package_name
if sudo(cmd).failed:
abort("Failed to download %s" % package_name)
if sudo("dpkg -i %s" % package_name).failed:
abort("Failed to install %s" % package_name)
sudo("rm -rf %s" % package_name)
# install python-packages
with cd(current_dir):
if sudo("pip install -r requirements.txt").failed:
abort("Failed to install python libraries")
#copy heatmap to webserver directory and set permissions
if sudo("mkdir /var/www/html/heatmap").failed:
print("Failed to mkdir /var/www/html/heatmap")
if sudo("chmod 777 /var/www/html/heatmap").failed:
print("Failed to set perms for heatmap directory")
with cd(current_dir):
sudo("cp -r heatmap/* /var/www/html/heatmap")
# check bios.bin file
bios_files = [
"/usr/share/qemu/bios.bin", "/usr/share/qemu/vgabios-cirrus.bin"
]
for bios_file in bios_files:
if not os.path.exists(bios_file):
filename = os.path.basename(bios_file)
sudo("ln -s /usr/share/seabios/%s %s" % (filename, bios_file))
# disable libvirtd from appArmor to enable custom KVM
if sudo("aa-complain /usr/sbin/libvirtd").failed:
abort("Failed to disable AppArmor for custom KVM")
# add current user to groups (optional)
username = env.get('user')
if sudo("adduser %s kvm" % username).failed:
abort("Cannot add user to kvm group")
if sudo("adduser %s libvirtd" % username).failed:
abort("Cannot add user to libvirtd group")
# Check fuse support:
# qemu-kvm changes the permission of /dev/fuse, so we revert back the
# permission. This bug is fixed from udev-175-0ubuntu26
# Please see https://bugs.launchpad.net/ubuntu/+source/udev/+bug/1152718
if sudo("chmod 1666 /dev/fuse").failed:
abort("Failed to enable fuse for the user")
if sudo("chmod 644 /etc/fuse.conf").failed:
abort("Failed to change permission of fuse configuration")
if sudo("sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf"):
abort("Failed to allow other user to access FUSE file")
# install cloudlet package
with cd(current_dir):
# remove previous build directory
with settings(hide('everything')):
sudo("rm -rf ./build")
sudo("pip uninstall --y elijah-provisioning")
# install python package
if sudo("python setup.py install").failed:
abort("cannot install cloudlet library")
# clean-up
with settings(hide('everything')):
sudo("rm -rf ./build")
sys.stdout.write("[SUCCESS] VM synthesis code is installed\n")
def create():
"""
Creates the environment needed to host the project.
The environment consists of: system locales, virtualenv, database, project
files, SSL certificate, and project-specific Python requirements.
"""
# Generate project locale
locale = env.locale.replace("UTF-8", "utf8")
with hide("stdout"):
if locale not in run("locale -a"):
sudo("locale-gen %s" % env.locale)
sudo("update-locale %s" % env.locale)
sudo("service postgresql restart")
run("exit")
# Create project path
run("mkdir -p %s" % env.proj_path)
# Set up virtual env
run("mkdir -p %s" % env.venv_home)
with cd(env.venv_home):
if exists(env.proj_name):
if confirm("Virtualenv already exists in host server: %s"
"\nWould you like to replace it?" % env.proj_name):
run("rm -rf %s" % env.proj_name)
else:
abort()
run("virtualenv %s" % env.proj_name)
# Upload project files
if env.deploy_tool in env.vcs_tools:
vcs_upload()
else:
rsync_upload()
# Create DB and DB user
pw = db_pass()
user_sql_args = (env.proj_name, pw.replace("'", "\'"))
user_sql = "CREATE USER %s WITH ENCRYPTED PASSWORD '%s';" % user_sql_args
psql(user_sql, show=False)
shadowed = "*" * len(pw)
print_command(user_sql.replace("'%s'" % pw, "'%s'" % shadowed))
psql("CREATE DATABASE %s WITH OWNER %s ENCODING = 'UTF8' "
"LC_CTYPE = '%s' LC_COLLATE = '%s' TEMPLATE template0;" %
(env.proj_name, env.proj_name, env.locale, env.locale))
# Set up SSL certificate
if not env.ssl_disabled:
conf_path = "/etc/nginx/conf"
if not exists(conf_path):
sudo("mkdir %s" % conf_path)
with cd(conf_path):
crt_file = env.proj_name + ".crt"
key_file = env.proj_name + ".key"
if not exists(crt_file) and not exists(key_file):
try:
crt_local, = glob(join("deploy", "*.crt"))
key_local, = glob(join("deploy", "*.key"))
except ValueError:
parts = (crt_file, key_file, env.domains[0])
sudo("openssl req -new -x509 -nodes -out %s -keyout %s "
"-subj '/CN=%s' -days 3650" % parts)
else:
upload_template(crt_local, crt_file, use_sudo=True)
upload_template(key_local, key_file, use_sudo=True)
# Install project-specific requirements
upload_template_and_reload("settings")
with project():
if env.reqs_path:
pip("-r %s/%s" % (env.proj_path, env.reqs_path))
pip("gunicorn setproctitle psycopg2 "
"django-compressor python-memcached")
# Bootstrap the DB
manage("createdb --noinput --nodata")
python(
"from django.conf import settings;"
"from django.contrib.sites.models import Site;"
"Site.objects.filter(id=settings.SITE_ID).update(domain='%s');" %
env.domains[0])
for domain in env.domains:
python("from django.contrib.sites.models import Site;"
"Site.objects.get_or_create(domain='%s');" % domain)
if env.admin_pass:
pw = env.admin_pass
user_py = ("from django.contrib.auth import get_user_model;"
"User = get_user_model();"
"u, _ = User.objects.get_or_create(username='******');"
"u.is_staff = u.is_superuser = True;"
"u.set_password('%s');"
"u.save();" % pw)
python(user_py, show=False)
shadowed = "*" * len(pw)
print_command(user_py.replace("'%s'" % pw, "'%s'" % shadowed))
return True
def commit_pull_and_deploy(tag):
# Clone the Docker configuration in a local temporary directory
local_tmpdir = tempfile.mkdtemp(prefix='fab-deploy')
local_compose_file = os.path.join(
local_tmpdir, env.docker_git_compose_file)
with lcd(local_tmpdir):
local("git clone --quiet --depth=1 --branch='{}' '{}' .".format(
env.docker_git_branch, env.docker_git_repo)
)
# Update the image tag used by Docker Compose
image_name = '{}:{}'.format(DOCKER_HUB_REPO, tag)
updated_compose_image = False
with open(local_compose_file, 'r') as f:
compose_file_lines = f.readlines()
with open(local_compose_file, 'w') as f:
for line in compose_file_lines:
matches = re.match(DOCKER_COMPOSE_IMAGE_UPDATE_PATTERN, line)
if not matches:
f.write(line)
continue
else:
# https://docs.python.org/2/library/os.html#os.linesep
f.write('{prefix}{image_name}\n'.format(
prefix=matches.group(1), image_name=image_name)
)
updated_compose_image = True
if not updated_compose_image:
raise Exception(
'Failed to update image to {} in Docker Compose '
'configuration'.format(image_name)
)
# Did we actually make a change?
if local('git diff', capture=True):
# Commit the change
local("git add '{}'".format(local_compose_file))
local("git commit -am 'Upgrade {service} to {tag}'".format(
service=SERVICE_NAME, tag=tag)
)
# Push the commit
local('git show')
response = prompt(
'OK to push the above commit to {} branch of {}? (y/n)'.format(
env.docker_git_branch, env.docker_git_repo)
)
if response != 'y':
abort('Push cancelled')
local("git push origin '{}'".format(env.docker_git_branch))
# Make a note of the commit to verify later that it's pulled to the
# remote server
pushed_config_commit = local("git show --no-patch", capture=True)
# Deploy to the remote server
with cd(env.docker_config_path):
run('git pull')
pulled_config_commit = run_no_pty("git show --no-patch")
if pulled_config_commit != pushed_config_commit:
raise Exception(
'The configuration commit on the remote server does not match '
'what was pushed locally. Please make sure {} is checked out '
'on the remote server.'.format(env.docker_git_branch)
)
run_no_pty("{doco} pull '{service}'".format(
doco=env.docker_compose_command, service=SERVICE_NAME)
)
run("{doco} up -d".format(doco=env.docker_compose_command))
def test():
with settings(warn_only=True):
result = local("python ./manage.py test apps.blog", capture=True)
if result.failed and not confirm("Test failed, Continue anyway"):
abort("aborting at user request")
def _check_rollback_to():
if not env.release_path:
abort('No release to rollback')
def run_experiment(test_id='', test_id_pfx='', *args, **kwargs):
do_init_os = kwargs.get('do_init_os', '1')
ecn = kwargs.get('ecn', '0')
tcp_cc_algo = kwargs.get('tcp_cc_algo', 'default')
duration = kwargs.get('duration', '')
if duration == '':
abort('No experiment duration specified')
# create sub directory for test id prefix
mkdir_p(test_id_pfx)
# remove <test_id>* files in <test_id_pfx> directory if exists
file_pattern = test_id_pfx + "/" + test_id + "_*"
for f in glob.glob(file_pattern):
os.remove(f)
# log experiment in started list
local('echo "%s" >> experiments_started.txt' % test_id)
puts('\n[MAIN] Starting experiment %s \n' % test_id)
tftpboot_dir = ''
try:
tftpboot_dir = config.TPCONF_tftpboot_dir
except AttributeError:
pass
# initialise
if tftpboot_dir != '' and do_init_os == '1':
execute(get_host_info,
netint='0',
hosts=config.TPCONF_router + config.TPCONF_hosts)
execute(init_os_hosts, file_prefix=test_id_pfx,
local_dir=test_id_pfx) # reboot
clear_type_cache() # clear host type cache
disconnect_all() # close all connections
time.sleep(30) # give hosts some time to settle down (after reboot)
# initialise topology
try:
switch = ''
port_prefix = ''
port_offset = 0
try:
switch = config.TPCONF_topology_switch
port_prefix = config.TPCONF_topology_switch_port_prefix
port_offset = config.TPCONF_topology_switch_port_offset
except AttributeError:
pass
if config.TPCONF_config_topology == '1' and do_init_os == '1':
# we cannot call init_topology directly, as it is decorated with
# runs_once. in experiment.py we have empty host list whereas if we
# run init_topology from command line we have the -H host list. executing
# an runs_once task with empty host list (hosts set in execute call), it
# will only be executed for the first host, which is not what we
# want. in contrast if we have a host list in context, execute will be
# executed once for each host (hence we need runs_once when called from
# the command line).
# sequentially configure switch
execute(init_topology_switch,
switch,
port_prefix,
port_offset,
hosts=config.TPCONF_hosts)
# configure hosts in parallel
execute(init_topology_host, hosts=config.TPCONF_hosts)
except AttributeError:
pass
file_cleanup(test_id_pfx) # remove any .start files
execute(get_host_info,
netmac='0',
hosts=config.TPCONF_router + config.TPCONF_hosts)
execute(sanity_checks)
execute(init_hosts, *args, **kwargs)
# first is the legacy case with single router and single queue definitions
# second is the multiple router case with several routers and several queue
# definitions
if isinstance(config.TPCONF_router_queues, list):
# start queues/pipes
config_router_queues(config.TPCONF_router_queues, config.TPCONF_router,
**kwargs)
# show pipe setup
execute(show_pipes, hosts=config.TPCONF_router)
elif isinstance(config.TPCONF_router_queues, dict):
for router in config.TPCONF_router_queues.keys():
# start queues/pipes for router r
config_router_queues(config.TPCONF_router_queues[router], [router],
**kwargs)
# show pipe setup
execute(show_pipes, hosts=[router])
# log config parameters
execute(log_config_params,
file_prefix=test_id,
local_dir=test_id_pfx,
hosts=['MAIN'],
*args,
**kwargs)
# log host tcp settings
execute(log_host_tcp,
file_prefix=test_id,
local_dir=test_id_pfx,
hosts=['MAIN'],
*args,
**kwargs)
# start all loggers
execute(start_loggers,
file_prefix=test_id,
local_dir=test_id_pfx,
remote_dir=config.TPCONF_remote_dir)
# Start broadcast ping and loggers (if enabled)
try:
if config.TPCONF_bc_ping_enable == '1':
# for multicast need IP of outgoing interface
# which is router's control interface
use_multicast = socket.gethostbyname(
config.TPCONF_router[0].split(':')[0])
# get configured broadcast or multicast address
bc_addr = ''
try:
bc_addr = config.TPCONF_bc_ping_address
except AttributeError:
# use default multicast address
bc_addr = '224.0.1.199'
execute(start_bc_ping_loggers,
file_prefix=test_id,
local_dir=test_id_pfx,
remote_dir=config.TPCONF_remote_dir,
bc_addr=bc_addr)
try:
bc_ping_rate = config.TPCONF_bc_ping_rate
except AttributeError:
bc_ping_rate = '1'
# start the broadcst ping on the first router
execute(start_bc_ping,
file_prefix=test_id,
local_dir=test_id_pfx,
remote_dir=config.TPCONF_remote_dir,
bc_addr=bc_addr,
rate=bc_ping_rate,
use_multicast=use_multicast,
hosts=[config.TPCONF_router[0]])
except AttributeError:
pass
# start traffic generators
sync_delay = 5.0
max_wait_time = sync_delay
start_time = datetime.datetime.now()
for t, c, v in sorted(config.TPCONF_traffic_gens, cmp=_cmp_timekeys):
try:
# delay everything to have synchronised start
next_time = float(t) + sync_delay
except ValueError:
abort('Traffic generator entry key time must be a float')
if next_time > max_wait_time:
max_wait_time = next_time
# add the kwargs parameter to the call of _param
v = re.sub("(V_[a-zA-Z0-9_-]*)", "_param('\\1', kwargs)", v)
# trim white space at both ends
v = v.strip()
if v[-1] != ',':
v = v + ','
# add counter parameter
v += ' counter="%s"' % c
# add file prefix parameter
v += ', file_prefix=test_id'
# add remote dir
v += ', remote_dir=\'%s\'' % config.TPCONF_remote_dir
# add test id prefix to put files into correct directory
v += ', local_dir=\'%s\'' % test_id_pfx
# we don't need to check for presence of tools inside start functions
v += ', check="0"'
# set wait time until process is started
now = datetime.datetime.now()
dt_diff = now - start_time
sec_diff = (dt_diff.days * 24 * 3600 + dt_diff.seconds) + \
(dt_diff.microseconds / 1000000.0)
if next_time - sec_diff > 0:
wait = str(next_time - sec_diff)
else:
wait = '0.0'
v += ', wait="' + wait + '"'
_nargs, _kwargs = eval('_args(%s)' % v)
execute(*_nargs, **_kwargs)
# print process list
print_proc_list()
# wait until finished (add additional 5 seconds to be sure)
total_duration = float(duration) + max_wait_time + 5.0
puts('\n[MAIN] Running experiment for %i seconds\n' % int(total_duration))
time.sleep(total_duration)
# shut everything down and get log data
execute(stop_processes, local_dir=test_id_pfx)
execute(log_queue_stats,
file_prefix=test_id,
local_dir=test_id_pfx,
hosts=config.TPCONF_router)
# log test id in completed list
local('echo "%s" >> experiments_completed.txt' % test_id)
# kill any remaining processes
execute(kill_old_processes,
hosts=config.TPCONF_router + config.TPCONF_hosts)
# done
puts('\n[MAIN] COMPLETED experiment %s \n' % test_id)
def init_tc_pipe(counter='1',
source='',
dest='',
rate='',
delay='',
rtt='',
loss='',
queue_size='',
queue_size_mult='1.0',
queue_disc='',
queue_disc_params='',
bidir='0',
attach_to_queue=''):
# compatibility with FreeBSD
if queue_disc == 'fifo':
# pfifo is the default for HTB classes
queue_disc = 'pfifo'
queue_size = str(queue_size)
if queue_size.lower() == 'bdp':
_rate = rate.replace('kbit', '000')
_rate = _rate.replace('mbit', '000000')
if rtt == '':
rtt = str(2 * int(delay))
if queue_disc == 'pfifo' or queue_disc == 'codel' or \
queue_disc == 'fq_codel' or queue_disc == 'pie':
# queue size in packets
avg_packet = 600 # average packet size
queue_size = int(
float(_rate) * (float(rtt) / 1000.0) / 8 / avg_packet)
if queue_size_mult != '1.0':
queue_size = int(float(queue_size) * float(queue_size_mult))
if queue_size < 1:
queue_size = 1 # minimum 1 packet
queue_size = str(queue_size)
elif queue_disc == 'bfifo' or queue_disc == 'red':
# queue size in bytes
queue_size = int(float(_rate) * (float(rtt) / 1000.0) / 8)
if queue_size_mult != '1.0':
queue_size = int(float(queue_size) * float(queue_size_mult))
if queue_size < 2048:
queue_size = 2048 # minimum 2kB
queue_size = str(queue_size)
else:
abort('Can\'t specify \'bdp\' for queuing discipline %s' %
queue_disc)
# class/handle numbers
class_no = str(int(counter) + 0)
if attach_to_queue == '':
queue_class_no = class_no
else:
# if attach_to_queue is set we attach this to existing (previously
# configured pipe). this means packets will go through an existing htb
# and leaf qdisc, but a separate netem.
# so we can have different flows going through the same bottleneck
# queue, but with different emulated delays or loss rates
queue_class_no = attach_to_queue
netem_class_no = class_no
qdisc_no = str(int(counter) + 1000)
netem_no = str(int(counter) + 1000)
# disciplines: fq_codel, codel, red, choke, pfifo, pfifo_fast (standard
# magic), pie (only as patch), ...
if queue_disc == '':
queue_disc = 'pfifo'
# for pie we need to make sure the kernel module is loaded (for kernel pre
# 3.14 only, for new kernels it happens automatically via tc use!)
if queue_disc == 'pie':
with settings(warn_only=True):
run('modprobe pie')
if rate == '':
rate = '1000mbit'
if queue_size == '':
# set default queue size to 1000 packet (massive but default for e.g.
# codel)
queue_size = '1000'
if loss != '':
# convert to percentage
loss = str(float(loss) * 100)
interfaces = get_netint_cached(env.host_string, int_no=-1)
# our approach works as follows:
# - shaping, aqm and delay/loss emulation is done on egress interface
# (as usual)
# - use htb qdisc for rate limiting with the aqm qdisc (e.g. pfifo, codel)
# as leave node
# - after shaping and aqm, emulate loss and delay with netem
# - for each "pipe" we setup a new class on all (two) interfaces
# - if pipes are unidirectional a class is only used on one of the two ifaces;
# otherwise it is used on both interfaces (XXX could optimise the
# unidirectional case and omit unused pipes)
# - traffic flow is as follows:
# 1. packets are marked by iptables in mangle table POSTROUTING hook
# depending on defined source/dest (unique mark for each pipe)
# 2. marked packets are classified into appropriate class (1-1 mapping
# between marks and classes) and redirected to pseudo interface
# 3. pseudo interface does the shaping with htb and aqm (leaf qdisc)
# 4. packets go back to actual interface
# 5. actual interface does network emulation (delay/loss), here htb is set to
# max rate (1Gbps) and pfifo is used (effectively no shaping or aqm here)
# note that according to my information the htb has a build-in buffer of 1
# packet as well (cannot be changed)
cnt = 0
for interface in interfaces:
pseudo_interface = 'ifb' + str(cnt)
# config rate limiting on pseudo interface
config_tc_cmd = 'tc class add dev %s parent 1: classid 1:%s htb rate %s ceil %s' % \
(pseudo_interface, queue_class_no, rate, rate)
if attach_to_queue == '':
run(config_tc_cmd)
# config queuing discipline and buffer limit on pseudo interface
config_tc_cmd = 'tc qdisc add dev %s parent 1:%s handle %s: %s limit %s %s' % \
(pseudo_interface,
queue_class_no,
qdisc_no,
queue_disc,
queue_size,
queue_disc_params)
if attach_to_queue == '':
run(config_tc_cmd)
# configure filter to classify traffic based on mark on pseudo device
config_tc_cmd = 'tc filter add dev %s protocol ip parent 1: ' \
'handle %s fw flowid 1:%s' % (
pseudo_interface, class_no, queue_class_no)
run(config_tc_cmd)
# configure class for actual interface with max rate
config_tc_cmd = 'tc class add dev %s parent 1: classid 1:%s ' \
'htb rate 1000mbit ceil 1000mbit' % \
(interface, netem_class_no)
run(config_tc_cmd)
# config netem on actual interface
config_tc_cmd = 'tc qdisc add dev %s parent 1:%s handle %s: ' \
'netem limit 1000' % (
interface, netem_class_no, netem_no)
if delay != "":
config_tc_cmd += " delay %sms" % delay
if loss != "":
config_tc_cmd += " loss %s%%" % loss
run(config_tc_cmd)
# configure filter to redirect traffic to pseudo device first and also
# classify traffic based on mark after leaving the pseudo interface traffic
# will go back to actual interface
config_tc_cmd = 'tc filter add dev %s protocol ip parent 1: handle %s ' \
'fw flowid 1:%s action mirred egress redirect dev %s' % \
(interface, class_no, netem_class_no, pseudo_interface)
run(config_tc_cmd)
cnt += 1
# filter on specific ips
config_it_cmd = 'iptables -t mangle -A POSTROUTING -s %s -d %s -j MARK --set-mark %s' % \
(source, dest, class_no)
run(config_it_cmd)
if bidir == '1':
config_it_cmd = 'iptables -t mangle -A POSTROUTING -s %s -d %s -j MARK --set-mark %s' % \
(dest, source, class_no)
run(config_it_cmd)
def test():
with settings(warn_only=True):
result = local('./manage.py test hmf_finder', capture=True)
if result.failed and not confirm("Tests failed. Continue anyway?"):
abort("Aborting at user request.")
def adjust_timestamps(test_id='',
file_name='',
host_name='',
sep=' ',
out_dir=''):
"Adjust timestamps in data file based on observed clock offsets"
# out_dir is the user-specified out_dir we pass on to get_clock_offsets()
if len(out_dir) > 0 and out_dir[-1] != '/':
out_dir += '/'
# out_dirname is the directory where the clockoffset file will be
if out_dir == '' or out_dir[0] != '/':
out_dirname = os.path.dirname(file_name)
else:
out_dirname = out_dir
if out_dirname[-1] != '/':
out_dirname += '/'
# clock offset file name
offs_fname = out_dirname + test_id + CLOCK_OFFSET_FILE_EXT
# new file name
new_fname = file_name + DATA_CORRECTED_FILE_EXT
#print(offs_fname)
if not os.path.isfile(offs_fname):
execute(get_clock_offsets, test_id=test_id, out_dir=out_dir)
if not os.path.isfile(offs_fname):
# give up and just make a copy of the existing data, so we have a file
# with .tscorr extension
#warn('Cannot generate clock offset file, using uncorrected timestamps '
# 'for experiment %s' % test_id)
#local('cp %s %s' % (file_name, new_fname))
# abort so we are on the safe side, user needs to fix or rerun with
# ts_corerct=0
abort('Cannot generate clock offset file for experiment %s' % test_id)
return new_fname
host_times = []
last_offs = 0.0
try:
with open(offs_fname) as f:
offs_lines = f.readlines()
# find column (note # is first column in first row
host_col = -1
for col in offs_lines[0].rstrip().split(' '):
if col == host_name:
break
host_col += 1
#print(host_name)
#print(host_col)
for line in offs_lines[1:]:
line = line.rstrip()
ref_time = line.split(' ')[0]
offs = line.split(' ')[host_col]
# if we have no data our offset for correction will be the
# last offset != zero otherwise it will be the offset measured
if offs == 'NA':
offs = last_offs
else:
offs = float(offs)
last_offs = offs
# XXX instead of using the instantenous offset values we may
# want to do something better in the future, such as using
# a weighted moving average etc.
host_times.append((ref_time, offs))
except IOError:
abort('Cannot open file %s' % offs_fname)
reader = csv.reader(open(file_name, 'r'), delimiter=sep)
fout = open(new_fname, 'w')
# index to curr_ref_time
curr = 0
for line in reader:
time = line[0]
# find the right time, we assume here that each offset is
# valid from the time it was observed until the time the next
# offset is observed
while host_times[curr][0] < time or host_times[curr][0] == 'NA':
curr += 1
if curr > 0 and host_times[curr - 1][0] != 'NA':
curr -= 1
new_time = float(time) - host_times[curr][1]
fout.write('{0:.6f}'.format(new_time) + sep)
fout.write(sep.join(line[1:]))
fout.write('\n')
fout.close()
return new_fname
def get_netmac(internal_int='0'):
"Get MAC address for external/ctrl network interface"
# not so easy to get local mac address for all different OS
# use following approach:
# get all non-router macs via the router using arp -a
# get router mac using ifconfig
# if host is a vm which we access via localhost then use ifconfig not arp
# method
# XXX windows: getmac, ipconfig /all
host_string = env.host_string
# if we have a port then strip off port
if host_string.find(':') > -1:
host_string = host_string.split(':')[0]
if host_string == 'localhost':
host_string = '127.0.0.1'
if host_string in config.TPCONF_router or host_string == '127.0.0.1':
# get MAC of router
htype = get_type_cached(env.host_string)
# complicated awk code to get pairs of ip and mac addresses from
# ifconfig
if htype == 'FreeBSD':
macips = run(
'ifconfig | awk \'/ether / { printf("%s ", $0); next } 1\' | '
+ 'grep ether | grep "inet " | ' +
'awk \'{ printf("%s %s\\n", $2, $4) }\'',
shell=False)
elif htype == 'Linux':
macips = run(
'ifconfig | awk \'/HWaddr / { printf("%s ", $0); next } 1\' | '
+ 'grep HWaddr | grep "inet " | ' +
'awk \'{ printf("%s %s\\n", $5, $7) }\' | sed -e "s/addr://"')
else:
abort("Can't determine MAC address for OS %s" % htype)
ip_mac_map = {}
for line in macips.split('\n'):
a = line.split(' ')
ip_mac_map.update({a[1].strip(): a[0].strip()})
# print(ip_mac_map)
# check if it looks like a name
if not re.match('[0-9.]+', host_string):
# try dns and assume we get an answer
ip = socket.gethostbyname(host_string)
else:
ip = host_string
if ip != '127.0.0.1':
mac = ip_mac_map.get(ip)
else:
# guess it's the first NIC
# XXX should return MAC based on router IP, not simply the first
mac = ip_mac_map.get(ip_mac_map.keys()[0])
else:
# get MAC of non-router
if internal_int == '0':
host_string = env.host_string
else:
host_string = config.TPCONF_host_internal_ip.get(
env.host_string, '')[0]
mac = execute(_get_netmac,
host_string.split(':')[0],
hosts=[config.TPCONF_router[0]])[config.TPCONF_router[0]]
return mac.lower()
def _require_root():
"""Check if running as root."""
with hide("running"):
if local("whoami", capture=True) != "root":
abort("You must be root.")
def get_clock_offsets(exp_list='experiments_completed.txt',
test_id='',
pkt_filter='',
baseline_host='',
out_dir=''):
"Get clock offsets for all hosts"
if len(out_dir) > 0 and out_dir[-1] != '/':
out_dir += '/'
if test_id == '':
try:
with open(exp_list) as f:
test_id_arr = f.readlines()
except IOError:
abort('Cannot open file %s' % exp_list)
else:
test_id_arr = test_id.split(';')
if len(test_id_arr) == 0 or test_id_arr[0] == '':
abort('Must specify test_id parameter')
# specify complete tcpdump parameter list
tcpdump_filter = '-tt -r - -n ' + pkt_filter
for test_id in test_id_arr:
test_id = test_id.rstrip()
# first find tcpdump files
tcpdump_files = get_testid_file_list('', test_id, '_ctl.dmp.gz', '')
if len(tcpdump_files) == 0:
warn('No tcpdump files for control interface for %s' % test_id)
continue
# if we have tcpdumps for control interface we can assume broadcast ping
# was enabled
dir_name = os.path.dirname(tcpdump_files[0])
# then look for tpconf_vars.log.gz file in that directory
var_file = local('find -L %s -name "*tpconf_vars.log.gz"' % dir_name,
capture=True)
bc_addr = ''
router = ''
if len(var_file) > 0:
# new approach without using config.py
# XXX no caching here yet, assume we only generate clockoffset file once
# per experiment
# unzip archived file
local('gzip -cd %s > %s' % (var_file, TMP_CONF_FILE))
# load the TPCONF_variables into oldconfig
oldconfig = imp.load_source('oldconfig', TMP_CONF_FILE)
# remove temporary unzipped file
try:
os.remove(TMP_CONF_FILE)
os.remove(TMP_CONF_FILE +
'c') # remove the compiled file as well
except OSError:
pass
try:
bc_addr = oldconfig.TPCONF_bc_ping_address
except AttributeError:
pass
router_name = oldconfig.TPCONF_router[0].split(':')[0]
else:
# old approach using config.py
try:
bc_addr = config.TPCONF_bc_ping_address
except AttributeError:
pass
router_name = config.TPCONF_router[0].split(':')[0]
if bc_addr == '':
# assume default multicast address
bc_addr = '224.0.1.199'
# specify complete tcpdump parameter list
if pkt_filter != '':
tcpdump_filter = '-tt -r - -n ' + pkt_filter
else:
tcpdump_filter = '-tt -r - -n ' + 'icmp and dst host ' + bc_addr
if baseline_host == '':
baseline_host = router_name
#
# now read timestamps from each host's tcpdump
#
# map of host names (or IPs) and sequence numbers to timestamps
host_times = {}
for tcpdump_file in tcpdump_files:
host = local(
'echo %s | sed "s/.*_\([a-z0-9\.]*\)_ctl.dmp.gz/\\1/"' %
tcpdump_file,
capture=True)
host_times[host] = {}
#print(host)
#print(host_times)
# We pipe gzcat through to tcpdump. Note, since tcpdump exits early
# (due to "-c num_samples") gzcat's pipe will collapse and gzcat
# will complain bitterly. So we dump its stderr to stderrhack.
init_zcat = Popen(['zcat ' + tcpdump_file],
stdin=None,
stdout=PIPE,
stderr=stderrhack,
shell=True)
init_tcpdump = Popen(['tcpdump ' + tcpdump_filter],
stdin=init_zcat.stdout,
stdout=PIPE,
stderr=stderrhack,
shell=True)
for line in init_tcpdump.stdout.read().splitlines():
_time = line.split(" ")[0]
_seq = int(line.split(" ")[11].replace(',', ''))
host_times[host][_seq] = _time
#print(host_times)
# get time differences and get host list
diffs = {}
ref_times = {}
host_str = ''
host_list = sorted(host_times.keys())
# getting hosts from the config is problematic if different
# experiments with different configs in same directory
#host_list = sorted(config.TPCONF_router + config.TPCONF_hosts)
for host in host_list:
host_str += ' ' + host
if host not in host_times:
continue
for seq in sorted(host_times[host].keys()):
if seq not in diffs:
diffs[seq] = {}
if baseline_host in host_times and seq in host_times[
baseline_host]:
diffs[seq][host] = float(host_times[host][seq]) - \
float(host_times[baseline_host][seq])
ref_times[seq] = host_times[baseline_host][seq]
else:
# this should only happen if TPCONF_router was
# modified
warn('Cant find baseline host %s timestamp data' %
baseline_host)
diffs[seq][host] = None
ref_times[seq] = None
#print(diffs)
if out_dir == '' or out_dir[0] != '/':
dir_name = os.path.dirname(tcpdump_files[0])
out_dir = dir_name + '/' + out_dir
mkdir_p(out_dir)
out_name = out_dir + test_id + CLOCK_OFFSET_FILE_EXT
# write table of offsets (rows = time, cols = hosts)
f = open(out_name, 'w')
f.write('# ref_time' + host_str + '\n')
for seq in sorted(diffs.keys()):
if ref_times[seq] is not None:
f.write(ref_times[seq])
else:
# this case should not never happen
continue
f.write(' ')
for host in host_list:
if host in diffs[seq] and diffs[seq][host] is not None:
f.write('{0:.6f}'.format(diffs[seq][host]))
else:
f.write('NA')
if host != host_list[-1]:
f.write(' ')
f.write('\n')
f.close()
def forcemigrate(app=None):
"""Force migrations to apply for a given app."""
if app is None:
abort("No app name given.")
local("./manage.py migrate {} --fake".format(app))
local("./manage.py migrate {}".format(app))
def handle(self, noinput, debug, remote='', *args, **options):
# Load server config from project
config, remote = load_config(env,
remote,
config_user='******',
debug=debug)
# Set local project path
local_project_path = django_settings.SITE_ROOT
print(os.getenv('DJANGO_SETTINGS_MODULE'))
if django_settings.DEBUG:
abort(
"You're currently using your local settings file, you need use production instead.\n"
"To use production settings pass `--settings={}` to the deploy command."
.format(
os.getenv('DJANGO_SETTINGS_MODULE').replace(
'.local', '.production')))
# Change into the local project folder
with hide('output', 'running', 'warnings'):
with lcd(local_project_path):
# Get the Git repo URL.
remotes = local('git remote', capture=True).split('\n')
if len(remotes) == 1:
git_remote = local('git config --get remote.{}.url'.format(
remotes[0]),
capture=True)
else:
def validate_choice(choice):
if choice in remotes:
return choice
raise Exception('That is not a valid choice.')
choice = prompt('Which Git remote would you like to use?',
validate=validate_choice)
git_remote = local(
'git config --get remote.{}.url'.format(choice),
capture=True)
# Is this a bitbucket repo?
is_bitbucket_repo = '*****@*****.**' in git_remote
is_github_repo = 'github.com' in git_remote
if is_bitbucket_repo:
bb_regex = re.match(r'git@bitbucket\.org:(.+)/(.+)\.git',
git_remote)
if bb_regex:
bitbucket_account = bb_regex.group(1)
bitbucket_repo = bb_regex.group(2)
else:
raise Exception(
'Unable to determine Bitbucket details.')
elif is_github_repo:
gh_regex = re.match(
r'(?:git@|https://)github.com[:/]([\w\-]+)/([\w\-.]+)\.git$',
git_remote)
if gh_regex:
github_account = gh_regex.group(1)
github_repo = gh_regex.group(2)
else:
raise Exception('Unable to determine Github details.')
else:
raise Exception(
'Unable to determine Git host from remote URL: {}'.
format(git_remote))
project_folder = local_project_path.replace(
os.path.abspath(os.path.join(local_project_path, '..')) +
'/', '')
with settings(warn_only=True):
if local('[[ -e ../requirements.txt ]]').return_code:
raise Exception("No requirements.txt")
# Compress the domain names for nginx
domain_names = " ".join(django_settings.ALLOWED_HOSTS)
# Use the site domain as a fallback domain
fallback_domain_name = django_settings.SITE_DOMAIN
if not noinput:
fallback_domain_name = prompt('What should the default domain be?',
default=fallback_domain_name)
domain_names = prompt(
'Which domains would you like to enable in nginx?',
default=domain_names)
else:
print('Default domain: ', fallback_domain_name)
print('Domains to be enabled in nginx: ', domain_names)
# If the domain is pointing to the droplet already, we can setup SSL.
setup_ssl_for = [
domain_name for domain_name in domain_names.split(' ')
if local('dig +short {}'.format(domain_name), capture=True) ==
remote['server']['ip']
]
if not setup_ssl_for:
abort(
"Sorry, it's $CURRENT_YEAR, you need to use SSL. Please update the domain DNS to point to {}."
.format(remote['server']['ip']))
for domain_name in domain_names.split(' '):
if domain_name not in setup_ssl_for:
print('SSL will not be configured for {}'.format(domain_name))
# Override username (for DO hosts).
if env.user == 'deploy':
env.user = '******'
# Print some information for the user
print('')
print('Project: {}'.format(project_folder))
print('Server IP: {}'.format(env.host_string))
print('Server user: {}'.format(env.user))
print('')
# Get BitBucket / Github details
if is_bitbucket_repo:
if os.environ.get('BITBUCKET_USERNAME', False) and os.environ.get(
'BITBUCKET_PASSWORD', False):
bitbucket_username = os.environ.get('BITBUCKET_USERNAME')
bitbucket_password = os.environ.get('BITBUCKET_PASSWORD')
else:
bitbucket_username = prompt(
'Please enter your BitBucket username:'******'Please enter your BitBucket password: '******'GITHUB_TOKEN', False):
github_token = os.environ.get('GITHUB_TOKEN')
else:
github_token = prompt(
'Please enter your Github token (obtained from https://github.com/settings/tokens):'
)
circle_token = os.environ.get('CIRCLE_TOKEN', None)
print("")
# Create session_files
session_files = {
'gunicorn_start':
NamedTemporaryFile(mode='w+', delete=False),
'supervisor_config':
NamedTemporaryFile(mode='w+', delete=False),
'memcached_supervisor_config':
NamedTemporaryFile(mode='w+', delete=False),
'nginx_site_config':
NamedTemporaryFile(mode='w+', delete=False),
'apt_periodic':
NamedTemporaryFile(mode='w+', delete=False),
'certbot_cronjob':
NamedTemporaryFile(mode='w+', delete=False),
}
# Parse files
session_files['gunicorn_start'].write(
render_to_string(
'gunicorn_start', {
'project': project_folder,
'settings': remote['server'].get('settings_file',
'production')
}))
session_files['gunicorn_start'].close()
session_files['supervisor_config'].write(
render_to_string('supervisor_config', {'project': project_folder}))
session_files['supervisor_config'].close()
session_files['memcached_supervisor_config'].write(
render_to_string('memcached_supervisor_config',
{'project': project_folder}))
session_files['memcached_supervisor_config'].close()
session_files['nginx_site_config'].write(
render_to_string(
'nginx_site_config', {
'project': project_folder,
'domain_names': domain_names,
'fallback_domain_name': fallback_domain_name
}))
session_files['nginx_site_config'].close()
session_files['apt_periodic'].write(render_to_string('apt_periodic'))
session_files['apt_periodic'].close()
session_files['certbot_cronjob'].write(
render_to_string('certbot_cronjob'))
session_files['certbot_cronjob'].close()
# Define the locales first.
locale_tasks = [
{
'title':
'Modify the locales config',
'command':
'; '.join([
"sed -i 's/^# en_GB.UTF-8/en_GB.UTF-8/' /etc/locale.gen", # Uncomment the GB line
]),
},
{
'title': 'Generate locales',
'command': 'locale-gen --purge',
},
{
'title': 'Modify default locales',
'command': "sed -i 's/en_US/en_GB/' /etc/default/locale",
},
{
'title':
'Reconfigure locales',
'command':
'LANG=en_GB.UTF-8 dpkg-reconfigure -f noninteractive locales',
},
]
run_tasks(env, locale_tasks)
# Check if optional packages are defined in the config.
optional_packages = {}
if 'optional_packages' in config:
optional_packages = config['optional_packages']
python_version_full = remote['server'].get('python_version', '3')
python_version = python_version_full[0]
pip_command = 'pip{}'.format(3 if python_version == '3' else '')
python_command = 'python{}'.format(python_version_full)
# Define base tasks
base_tasks = [
# Add nginx and Let's Encrypt PPAs. We add them up here because an
# `apt-get update` is require for them to be truly added and that
# comes next.
{
'title': 'Add nginx PPA',
'command': 'add-apt-repository -y ppa:nginx/stable',
},
{
'title': "Add Let's Encrypt PPA",
'command': 'add-apt-repository -y ppa:certbot/certbot',
},
{
'title': 'Update apt cache',
'command': 'apt-get update',
},
{
'title': 'Upgrade everything',
'command': 'apt-get upgrade -y',
},
{
'title': 'Install unattended-upgrades',
'command': 'apt-get install -y unattended-upgrades',
},
{
'title':
"Install the base packages",
'command':
'apt-get install -y {}'.format(' '.join([
# Base requirements
'build-essential',
'git',
'ufw', # Installed by default on Ubuntu, not elsewhere
# Project requirements
'{}-dev'.format(python_command),
'python{}-pip'.format('3' if python_version ==
'3' else ''),
'apache2-utils', # Required for htpasswd
'python{}-passlib'.format(
'3' if python_version == '3' else ''
), # Required for generating the htpasswd file
'supervisor',
'libjpeg-dev',
'libffi-dev',
'libssl-dev', # Required for nvm.
'nodejs',
'memcached',
'fail2ban',
# Nginx things
'nginx',
'certbot',
'python-certbot-nginx',
# Postgres requirements
'postgresql',
'libpq-dev',
'python{}-psycopg2'.format(3 if python_version == '3' else
''), # TODO: Is this required?
# Required under Python 3.
'python3-venv' if python_version == '3' else '',
# Other
'libgeoip-dev'
if optional_packages.get('geoip', True) else '',
'libmysqlclient-dev' if optional_packages.get(
'mysql', True) else '',
'python3.6' if python_version_full == '3.6' else '',
'python3.6-dev' if python_version_full == '3.6' else '',
]))
},
{
'title':
'Adjust APT update intervals',
'fabric_command':
'put',
'fabric_args': [
session_files['apt_periodic'].name,
'/etc/apt/apt.conf.d/10periodic'
],
},
{
'title': 'Update pip',
'command': '{} install -U pip'.format(pip_command),
},
{
'title': 'Install virtualenv',
'command': '{} install virtualenv'.format(pip_command),
},
{
'title': 'Set the timezone to UTC',
'command': 'timedatectl set-timezone UTC',
}
]
if python_version_full == '3.6':
base_tasks.insert(
0, {
'title': 'Add Python 3.6 PPA',
'command':
'add-apt-repository -y ppa:jonathonf/python-3.6',
})
run_tasks(env, base_tasks)
# Configure swap
swap_tasks = [{
'title': 'Create a swap file',
'command': 'fallocate -l 4G /swapfile',
}, {
'title': 'Set permissions on swapfile to 600',
'command': 'chmod 0600 /swapfile'
}, {
'title': 'Format swapfile for swap',
'command': 'mkswap /swapfile',
}, {
'title': 'Add the file to the system as a swap file',
'command': 'swapon /swapfile',
}, {
'title':
'Write fstab line for swapfile',
'command':
"echo '/swapfile none swap sw 0 0' >> /etc/fstab",
}, {
'title': 'Change swappiness',
'command': 'sysctl vm.swappiness=10'
}, {
'title': 'Write swappiness to file',
'command': "echo 'vm.swappiness=10' >> /etc/sysctl.conf",
}, {
'title': 'Reduce cache pressure',
'command': 'sysctl vm.vfs_cache_pressure=50',
}, {
'title':
'Write cache pressure to file',
'command':
"echo 'vm.vfs_cache_pressure=50' >> /etc/sysctl.conf",
}]
# Check to see if we've already configured a swap file. This handles
# the case where the deploy command is being re-run.
cache_pressure = run('cat /proc/sys/vm/vfs_cache_pressure')
if cache_pressure != '50':
run_tasks(env, swap_tasks)
# Define SSH tasks
ssh_tasks = [{
'title': 'Create the application group',
'command': 'addgroup --system webapps',
}, {
'title':
'Add the application user',
'command':
'adduser --shell /bin/bash --system --disabled-password --ingroup webapps {name}'
.format(name=project_folder, ),
}, {
'title': "Add .ssh folder to application user's home directory",
'command': 'mkdir ~{}/.ssh'.format(project_folder),
}, {
'title':
'Generate SSH keys for application user',
'command':
"ssh-keygen -C application-server -f ~{}/.ssh/id_rsa -N ''".format(
project_folder, )
}, {
'title':
'Make the application directory',
'command':
'; '.join([
'mkdir -m 0775 -p /var/www/{project}',
'chown {project}:webapps /var/www/{project}',
]).format(project=project_folder, ),
}, {
'title':
'Check application user file permissions',
'command':
'&& '.join([
'chmod 0750 ~{project}',
'chmod 0700 ~{project}/.ssh',
'chmod 0600 ~{project}/.ssh/id_rsa',
'chmod 0644 ~{project}/.ssh/id_rsa.pub',
'chown -R {project}:webapps ~{project}',
]).format(project=project_folder, ),
}, {
'title':
'Add deploy user',
'command':
'adduser --shell /bin/bash --disabled-password --system --ingroup webapps deploy',
}, {
'title': "Add .ssh folder to deploy user's home directory",
'command': 'mkdir ~deploy/.ssh',
}, {
'title':
'Add authorized keys to deploy user',
'command':
'mv ~{}/.ssh/authorized_keys /home/deploy/.ssh/authorized_keys'.
format(env.user, ),
}, {
'title':
'Check deploy user file permissions',
'command':
'; '.join([
'chmod 0750 ~deploy',
'chmod 0700 ~deploy/.ssh',
'chmod 0644 ~deploy/.ssh/authorized_keys',
'chown -R deploy:webapps ~deploy',
]),
}, {
'title': 'Remove sudo group rights',
'command': "sed -i 's/^%sudo/# %sudo/' /etc/sudoers",
}, {
'title':
'Add deploy user to sudoers',
'command':
'echo "deploy ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/deploy',
}, {
'title':
'Ensure the deploy sudoers file has the correct permissions',
'command': 'chmod 0440 /etc/sudoers.d/deploy',
}, {
'title':
'Disallow root SSH access',
'command':
"sed -i 's/^PermitRootLogin yes/PermitRootLogin no/' /etc/ssh/sshd_config",
}, {
'title':
'Disallow password authentication',
'command':
"sed -i 's/^PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config",
}, {
'title': 'Restart SSH',
'command': 'service ssh restart',
}]
run_tasks(env, ssh_tasks)
# Define db tasks
db_tasks = [
{
'title':
'Create the application postgres role',
'command':
'su - postgres -c "createuser {name}"'.format(
name=remote['database']['name'], ),
},
{
'title':
'Ensure database is created',
'command':
'su - postgres -c "createdb {name} --encoding=UTF-8 --locale=en_GB.UTF-8 '
'--template=template0 --owner={owner} --no-password"'.format(
name=remote['database']['name'],
owner=remote['database']['user'],
),
},
{
'title':
'Ensure user has access to the database',
'command':
'su - postgres -c "psql {name} -c \'GRANT ALL ON DATABASE {name} TO {owner}\'"'
.format(
name=remote['database']['name'],
owner=remote['database']['user'],
),
},
{
'title':
'Ensure user does not have unnecessary privileges',
'command':
'su - postgres -c "psql {name} -c \'ALTER USER {owner} WITH NOSUPERUSER '
'NOCREATEDB\'"'.format(
name=remote['database']['name'],
owner=remote['database']['user'],
),
},
]
run_tasks(env, db_tasks)
# Get SSH Key from server
ssh_key = run('cat ~{}/.ssh/id_rsa.pub'.format(project_folder))
# Get the current SSH keys in the repo
if is_bitbucket_repo:
task_title = 'Checking bitbucket repository for an existing SSH key'
title_print(task_title, state='task')
try:
repo_ssh_keys = requests.get(
'https://bitbucket.org/api/1.0/repositories/{}/{}/deploy-keys/'
.format(
bitbucket_account,
bitbucket_repo,
),
auth=(bitbucket_username, bitbucket_password))
except:
title_print(task_title, state='failed')
exit()
title_print(task_title, state='succeeded')
task_title = 'Adding the SSH key to bitbucket'
if repo_ssh_keys.text.find(ssh_key) == -1:
title_print(task_title, state='task')
try:
requests.post(
'https://bitbucket.org/api/1.0/repositories/{}/{}/deploy-keys/'
.format(
bitbucket_account,
bitbucket_repo,
),
data=urlencode({
'label':
'Application Server ({})'.format(env.host_string),
'key':
ssh_key,
}),
auth=(bitbucket_username, bitbucket_password))
except Exception as e:
title_print(task_title, state='failed')
raise e
title_print(task_title, state='succeeded')
elif is_github_repo:
task_title = 'Adding the SSH key to Github'
title_print(task_title, state='task')
try:
response = requests.post(
'https://api.github.com/repos/{}/{}/keys'.format(
github_account, github_repo),
json={
'title':
'Application Server ({})'.format(env.host_string),
'key': ssh_key,
'read_only': True,
},
headers={'Authorization': 'token {}'.format(github_token)})
if debug:
print(response.text)
except Exception as e:
title_print(task_title, state='failed')
raise e
title_print(task_title, state='succeeded')
# Define git tasks
if is_bitbucket_repo:
git_url = '[email protected]:{}/{}.git'.format(
bitbucket_account,
bitbucket_repo,
)
elif is_github_repo:
git_url = '[email protected]:{}/{}.git'.format(
github_account,
github_repo,
)
git_tasks = [
{
'title':
'Add Github key to known hosts',
'command':
'ssh-keyscan -H github.com >> ~{project}/.ssh/known_hosts'.
format(project=project_folder, ),
},
{
'title':
'Setup the Git repo',
'command':
'cd /tmp; git clone {url} {project}'.format(
url=git_url,
project='/var/www/{}'.format(project_folder, )),
},
]
run_tasks(env, git_tasks, user=project_folder)
# Define static tasks
static_tasks = [
{
'title':
'Make the static directory',
'command':
'; '.join([
'mkdir -m 0775 -p {dir}',
'chown {project}:webapps {dir}',
]).format(
project=project_folder,
dir=django_settings.STATIC_ROOT,
),
},
{
'title':
'Make the media directory',
'command':
'; '.join([
'mkdir -m 0775 -p {dir}', 'chown {project}:webapps {dir}'
]).format(
project=project_folder,
dir=django_settings.MEDIA_ROOT,
),
},
]
run_tasks(env, static_tasks)
virtualenv_command = (
'virtualenv -p python{python_full} /var/www/{project}/.venv')
# Define venv tasks
venv_tasks = [
{
'title':
'Create the virtualenv',
'command':
virtualenv_command.format(
python_full=python_version_full,
project=project_folder,
),
},
# This shouldn't be necessary (we think we upgraded pip earlier)
# but it is - you'll get complaints about bdist_wheel without
# this.
{
'title':
'Upgrade pip inside the virtualenv',
'command':
'/var/www/{project}/.venv/bin/pip install --upgrade pip'.
format(project=project_folder, ),
},
]
run_tasks(env, venv_tasks, user=project_folder)
gunicorn_tasks = [
{
'title':
'Create the Gunicorn script file',
'fabric_command':
'put',
'fabric_args': [
session_files['gunicorn_start'].name,
'/var/www/{project}/gunicorn_start'.format(
project=project_folder, )
],
},
{
'title':
'Make the Gunicorn script file executable',
'command':
'chmod +x /var/www/{project}/gunicorn_start'.format(
project=project_folder, )
},
{
'title':
'chown the Gunicorn script file',
'command':
'chown {project}:webapps /var/www/{project}/gunicorn_start'.
format(project=project_folder, )
},
]
run_tasks(env, gunicorn_tasks)
log_tasks = [
{
'title':
'Create the application log file',
'command':
'; '.join([
'touch /var/log/gunicorn_supervisor.log',
'chown {}:webapps /var/log/gunicorn_supervisor.log'.format(
project_folder, ),
'chmod 0644 /var/log/gunicorn_supervisor.log',
]),
},
]
run_tasks(env, log_tasks)
requirement_tasks = [
{
# Check to see if we have a requirements file. Even though we check for
# it at the start of the deployment process, it hasn't necessarily been
# committed. So this check covers that.
'title':
"Install packages required by the Django app inside virtualenv",
'command':
'if [ -f /var/www/{project}/requirements.txt ]; then /var/www/{project}/.venv/bin/pip '
'install -r /var/www/{project}/requirements.txt; fi'.format(
project=project_folder, ),
},
{
'title':
'Make sure Gunicorn is installed',
'command':
'/var/www/{project}/.venv/bin/pip install gunicorn'.format(
project=project_folder, ),
},
]
run_tasks(env, requirement_tasks, user=project_folder)
# Define nginx tasks
nginx_tasks = [
{
'title':
'Ensure Nginx service is stopped', # This allows Certbot to run.
'command': 'service nginx stop',
},
{
'title':
'Create the Nginx configuration file',
'fabric_command':
'put',
'fabric_args': [
session_files['nginx_site_config'].name,
'/etc/nginx/sites-available/{}'.format(project_folder, )
],
},
{
'title': 'Create the .htpasswd file',
'command': 'htpasswd -c -b /etc/nginx/htpasswd onespace media',
},
{
'title': 'Ensure that the default site is disabled',
'command': 'rm /etc/nginx/sites-enabled/default',
},
{
'title':
'Ensure that the application site is enabled',
'command':
'ln -s /etc/nginx/sites-available/{project} /etc/nginx/sites-enabled/{project}'
.format(project=project_folder, ),
},
{
'title':
'Run certbot',
'command':
'certbot certonly --standalone -n --agree-tos --email [email protected] '
'--cert-name {} --domains {}'.format(fallback_domain_name,
','.join(setup_ssl_for)),
},
{
'title':
'Generate DH parameters (this may take a little while)',
'command': 'openssl dhparam -out /etc/ssl/dhparam.pem 2048',
},
{
'title': 'Ensure Nginx service is started',
'command': 'service nginx start',
},
{
'title':
'Configure certbot cronjob',
'fabric_command':
'put',
'fabric_args':
[session_files['certbot_cronjob'].name, '/etc/cron.d/certbot'],
},
{
'title':
'Ensure the certbot cronjob has the correct file permissions',
'command': 'chmod 0644 /etc/cron.d/certbot',
},
]
run_tasks(env, nginx_tasks)
# Configure the firewall.
firewall_tasks = [
{
'title': 'Allow SSH connections through the firewall',
'command': 'ufw allow OpenSSH'
},
{
'title': 'Allow SSH connections through the firewall',
'command': 'ufw allow "Nginx Full"'
},
{
'title': 'Enable the firewall, deny all other traffic',
'command':
'ufw --force enable', # --force makes it non-interactive
}
]
run_tasks(env, firewall_tasks)
# Define supervisor tasks
supervisor_tasks = [
{
'title':
'Create the Supervisor config file for the application',
'fabric_command':
'put',
'fabric_args': [
session_files['supervisor_config'].name,
'/etc/supervisor/conf.d/{}.conf'.format(project_folder, )
],
},
{
'title':
'Stopping memcached and removing from startup runlevels',
'command':
'; '.join([
'service memcached stop',
'systemctl disable memcached',
]),
},
{
'title':
'Create the Supervisor config file for memcached',
'fabric_command':
'put',
'fabric_args': [
session_files['memcached_supervisor_config'].name,
'/etc/supervisor/conf.d/memcached.conf'
],
},
{
'title': 'Re-read the Supervisor config files',
'command': 'supervisorctl reread',
},
{
'title':
'Update Supervisor to add the app in the process group',
'command': 'supervisorctl update',
},
]
run_tasks(env, supervisor_tasks)
# Define build system tasks
build_systems = {
"none": [],
"npm": [{
'title':
'Install nvm',
'command':
'cd /tmp; curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.32.1/install.sh'
' | bash',
}, {
'title':
'Activate nvm then install node and yarn',
'command':
'&& '.join([
'cd /var/www/{project}',
'. ~{project}/.nvm/nvm.sh',
'nvm install',
'npm install -g yarn',
'yarn',
'yarn run build',
]).format(project=project_folder, ),
}, {
'title':
'Ensure static folder exists in project',
'command':
'if [ ! -d "/var/www/{project}/{project}/static/" ]; then mkdir /var/www/{project}/{'
'project}/static/; fi'.format(project=project_folder, ),
}, {
'title':
'Collect static files',
'command':
'/var/www/{project}/.venv/bin/python /var/www/{project}/manage.py collectstatic '
'--noinput --link --settings={project}.settings.{settings}'.
format(
project=project_folder,
settings=remote['server'].get('settings_file',
'production'),
),
}]
}
run_tasks(env,
build_systems[remote['server'].get('build_system', 'none')],
user=project_folder)
# Delete files
for session_file in session_files:
os.unlink(session_files[session_file].name)
# Add the project to CircleCI
circle_tasks = [
{
'title':
'Create the CircleCI SSH key',
'fabric_command':
'local',
'fabric_args':
['mkdir dist; ssh-keygen -C circleci -f dist/id_rsa -N '
''],
},
{
'title':
'Follow the project on CircleCI',
'fabric_command':
'local',
'fabric_args': [
'curl -X POST https://circleci.com/api/v1.1/project/github/{github_account}/{'
'github_repo}/follow?circle-token={circle_token}'.format(
github_account=github_account,
github_repo=github_repo,
circle_token=circle_token,
)
]
},
{
'title':
'Add private SSH key to CircleCI',
'fabric_command':
'local',
'fabric_args': [
'curl -X POST --header "Content-Type: application/json" -d \'{{"hostname":"{'
'fallback_domain_name}","private_key":"{private_key}"}}\' '
'https://circleci.com/api/v1.1/project/github/{github_account}/{'
'github_repo}/ssh-key?circle-token={circle_token}'.format(
fallback_domain_name=fallback_domain_name,
private_key=open('dist/id_rsa', 'r').read(),
github_account=github_account,
github_repo=github_repo,
circle_token=circle_token,
)
]
},
{
'title':
'Add public key to server',
'command':
'echo "{}" >> ~deploy/.ssh/authorized_keys'.format(
open('dist/id_rsa.pub', 'r').read())
},
]
if circle_token and is_github_repo:
run_tasks(env, circle_tasks)
print(
'Initial application deployment has completed. You should now pushdb and pushmedia.'
)
