def load_db(filename=None):
"""Loads a dump into the database"""
env.box_dump_filename = filename
if not filename:
abort(red('Dump missing. "fab server.load_db:filename"', bold=True))
if not os.path.exists(filename):
abort(red('"%(box_dump_filename)s" does not exist.' % env, bold=True))
if not confirm(
"Completely replace the remote database" ' "%(box_database)s" (if it exists)?',
default=False,
):
return
env.box_remote_db = remote_env("DATABASE_URL")
if not env.box_remote_db:
abort(red("Unable to determine the remote DATABASE_URL", bold=True))
run('psql -c "DROP DATABASE IF EXISTS %(box_database)s"')
run(
"createdb %(box_database)s --encoding=UTF8 --template=template0"
" --owner=%(box_database)s"
)
run_local(
"cat %(box_dump_filename)s |" " ssh %(host_string)s psql %(box_remote_db)s"
)
run('psql %(box_database)s -c "REASSIGN OWNED BY admin ' ' TO %(box_database)s"')
def fetch_remote():
step("Updating git remote...")
env.box_idx = ("" if len(env.hosts) < 2 else "-%d" %
(env.hosts.index(env.host_string) + 1))
with settings(warn_only=True):
run_local("git fetch %(box_remote)s%(box_idx)s")
def deploy():
"""Checks whether everything is ready for deployment"""
step('Checking whether we are on the expected branch...')
with settings(warn_only=True), hide('everything'):
branch = run_local('git symbolic-ref -q --short HEAD', capture=True)
if not branch:
abort(red('No branch checked out, cannot continue.', bold=True))
if branch != env.box_branch:
puts(
red('Warning: The currently checked out branch is \'%s\', but'
' the environment \'%s\' runs on \'%s\'.' %
(branch, env.box_environment, env.box_branch)))
if not confirm('Continue deployment?', default=False):
abort('Aborting.')
step('\nChecking whether we are up to date...')
run_local('git push --dry-run origin %(box_branch)s')
execute('check.check')
execute('check.test')
with cd('%(box_domain)s'):
step('\nChecking for uncommitted changes on the server...')
result = run('git status --porcelain')
if result:
abort(red('Uncommitted changes detected, aborting deployment.'))
def dev(host="127.0.0.1", port=8000):
"""Runs the development server, SCSS watcher and backend services if they
are not running already"""
if host == "net":
host = own_ip()
puts(
green("Starting dev server on http://%s:%s/" % (host, port),
bold=True))
with tempfile.NamedTemporaryFile() as f:
# https://gist.github.com/jiaaro/b2e1b7c705022c2cf56888152a999f65
f.write("""\
trap "exit" INT TERM
trap "kill 0" EXIT
PYTHONWARNINGS=always venv/bin/python manage.py runserver 0.0.0.0:%(port)s &
HOST=%(host)s yarn run dev &
for job in $(jobs -p); do wait $job; done
""" % {
"port": port,
"host": host
})
f.flush()
run_local("bash %s" % f.name)
def fetch_remote():
step('Updating git remote...')
env.box_idx = '' if len(
env.hosts) < 2 else '-%d' % (env.hosts.index(env.host_string) + 1)
with settings(warn_only=True):
run_local('git fetch %(box_remote)s%(box_idx)s')
def pull():
# local.backend_tools without migrate step
run_local('venv/bin/pip install -r requirements.txt')
run_local('find . -name "*.pyc" -delete')
execute('local.pull_database')
execute('local.update')
execute('local.reset_passwords')
def add_remote():
""" Add a server repository as git remote """
env.box_idx = '' if len(
env.hosts) < 2 else '-%d' % (env.hosts.index(env.host_string) + 1)
with settings(warn_only=True):
run_local('git remote add -f %(box_remote)s%(box_idx)s'
' %(host_string)s:%(box_domain)s/')
def dump_db():
"""Dumps the database into the tmp/ folder"""
env.box_datetime = datetime.now().strftime('%Y-%m-%d-%s')
env.box_dump_filename = os.path.join(
os.getcwd(),
'tmp',
'%(box_database_local)s-local-%(box_datetime)s.sql' % env,
)
run_local('pg_dump -Ox %(box_database_local)s > %(box_dump_filename)s')
puts(green('\nWrote a dump to %(box_dump_filename)s' % env))
def create_virtualenv():
"""Creates the virtualenv and installs all Python requirements"""
run_local('python3 -m venv venv')
run_local('venv/bin/pip install -U wheel pip')
if os.path.exists('requirements.txt'):
run_local('venv/bin/pip install -r requirements.txt')
else:
run_local('venv/bin/pip install -U -r requirements-to-freeze.txt')
run_local('(printf "# AUTOGENERATED, DO NOT EDIT\n\n";'
'venv/bin/pip freeze -l'
# Until Ubuntu gets its act together:
' | grep -vE "(^pkg-resources)"'
') > requirements.txt')
def create_database():
"""Creates and migrates a Postgres database"""
if not confirm('Completely replace the local database'
' "%(box_database_local)s" (if it exists)?'):
return
run_local('dropdb --if-exists %(box_database_local)s')
run_local('createdb %(box_database_local)s'
' --encoding=UTF8 --template=template0')
with settings(warn_only=True):
run_local('venv/bin/python manage.py makemigrations elephantblog')
run_local('venv/bin/python manage.py makemigrations page')
run_local('venv/bin/python manage.py migrate')
def deploy():
"""Checks whether everything is ready for deployment"""
step("Checking whether we are on the expected branch...")
with settings(warn_only=True), hide("everything"):
branch = run_local("git symbolic-ref -q --short HEAD", capture=True)
if not branch:
abort(red("No branch checked out, cannot continue.", bold=True))
if branch != env.box_branch:
puts(
red("Warning: The currently checked out branch is '%s', but"
" the environment '%s' runs on '%s'." %
(branch, env.box_environment, env.box_branch)))
if not confirm("Continue deployment?", default=False):
abort("Aborting.")
execute("check.check")
execute("check.test")
with cd("%(box_domain)s"):
step("\nChecking for uncommitted changes on the server...")
result = run("git status --porcelain")
if result:
abort(red("Uncommitted changes detected, aborting deployment."))
def dump_db():
"""Dumps the database into the tmp/ folder"""
env.box_datetime = datetime.now().strftime('%Y-%m-%d-%s')
env.box_dump_filename = os.path.join(
os.getcwd(),
'tmp',
'%(box_database)s-%(box_environment)s-%(box_datetime)s.sql' % env,
)
env.box_remote_db = remote_env('DATABASE_URL')
if not env.box_remote_db:
abort(red('Unable to determine the remote DATABASE_URL', bold=True))
run_local('ssh %(host_string)s pg_dump -Ox %(box_remote_db)s'
' > %(box_dump_filename)s')
puts(green('\nWrote a dump to %(box_dump_filename)s' % env))
def dump_db():
"""Dumps the database into the tmp/ folder"""
env.box_datetime = datetime.now().strftime("%Y-%m-%d-%s")
env.box_dump_filename = os.path.join(
os.getcwd(),
"tmp",
"%(box_database)s-%(box_environment)s-%(box_datetime)s.sql" % env,
)
env.box_remote_db = remote_env("DATABASE_URL")
if not env.box_remote_db:
abort(red("Unable to determine the remote DATABASE_URL", bold=True))
run_local(
"ssh %(host_string)s pg_dump -Ox %(box_remote_db)s" " > %(box_dump_filename)s"
)
puts(green("\nWrote a dump to %(box_dump_filename)s" % env))
def mm():
"""Wrapper around the ``makemessages`` management command which excludes
dependencies (virtualenv, bower components, node modules)"""
run_local('venv/bin/python manage.py makemessages -a'
' -i app/cms'
' -i bower_components'
' -i node_modules'
' -i venv')
"""Also statici18n ``makemessages`` command will be executed"""
run_local('venv/bin/python manage.py makemessages -d djangojs -a'
' -e jsx,js'
' -i app/static/jsi18n'
' -i app/cms'
' -i app/templates/elephantblog'
' -i bower_components'
' -i node_modules'
' -i venv')
def dev(host='127.0.0.1', port=8000):
"""Runs the development server, SCSS watcher and backend services if they
are not running already"""
if host == 'net':
host = own_ip()
puts(
green('Starting dev server on http://%s:%s/' % (host, port),
bold=True))
jobs = [
lambda: run_local(
'venv/bin/python -Wonce manage.py runserver 0.0.0.0:%s' %
(port, ), ),
lambda: run_local('HOST=%s yarn run dev' % host),
]
jobs = [Process(target=j) for j in jobs]
[j.start() for j in jobs]
[j.join() for j in jobs]
def deploy(*args):
"""Deploys frontend and backend code to the server if the checking step
did not report any problems"""
execute('check.deploy')
step('\nCompiling static sources...')
run_local('yarn run prod')
step('\nPushing changes...')
run_local('git push origin %(box_branch)s')
step('\nDeploying new code on server...')
with cd('%(box_domain)s'):
run('git fetch')
run('git reset --hard origin/%(box_branch)s')
run('find . -name "*.pyc" -delete')
run('venv/bin/pip install -r requirements.txt')
run('venv/bin/python manage.py migrate --noinput')
step('\nUploading static files...')
rsync_project(
local_dir='static/',
remote_dir='%(box_domain)s/static/' % env,
delete=('clear' in args),
)
step('\nCollecting static files...')
with cd('%(box_domain)s'):
run('venv/bin/python manage.py collectstatic --noinput')
step('\nRunning system checks on server...')
with cd('%(box_domain)s'):
run('venv/bin/python manage.py check --deploy')
step('\nRestarting server process...')
for line in env['box_restart']:
run(line)
execute('git.fetch_remote')
def mm(language=None):
"""Wrapper around the ``makemessages`` management command which excludes
dependencies (virtualenv, bower components, node modules)"""
run_local(" ".join([
"venv/bin/python manage.py makemessages",
("-l %s" % language) if language else "-a",
"-i app/cms",
"-i bower_components",
"-i node_modules",
"-i venv",
]))
"""Also statici18n ``makemessages`` command will be executed"""
run_local(" ".join([
"venv/bin/python manage.py makemessages -d djangojs",
("-l %s" % language) if language else "-a",
"-e jsx,js",
"-i app/static/jsi18n",
"-i app/cms",
"-i app/templates/elephantblog",
"-i bower_components",
"-i node_modules",
"-i venv",
]))
def clone_repository():
puts(green('We need the repository to initialize the server.'))
with hide('running'):
output = run_local('git config remote.origin.url', capture=True)
repo = prompt('Repository', default=output)
if not repo:
puts(red('Cannot continue without a repository.'))
return 1
env.box_repository_url = repo
run('git clone -b %(box_branch)s %(box_repository_url)s %(box_domain)s')
execute('git.add_remote')
def clone_repository():
puts(green("We need the repository to initialize the server."))
with hide("running"):
output = run_local("git config remote.origin.url", capture=True)
repo = prompt("Repository", default=output)
if not repo:
puts(red("Cannot continue without a repository."))
return 1
env.box_repository_url = repo
run("git clone -b %(box_branch)s %(box_repository_url)s %(box_domain)s")
execute("git.add_remote")
def init_bitbucket():
username = default_env('BITBUCKET_USERNAME')
password = default_env('BITBUCKET_PASSWORD') # Should probably not be used
organization = default_env('BITBUCKET_ORGANIZATION')
if not username or not organization:
puts('Consider adding default values for BITBUCKET_USERNAME'
' and BITBUCKET_ORGANIZATION to ~/.box.env')
username = prompt('Username', default=username)
if not password:
password = getpass.getpass('Password ')
organization = prompt('Organization', default=organization)
repository = prompt('Repository', default=env.box_repository)
if not confirm('Initialize repository at https://bitbucket.org/%s/%s?' %
(organization, repository)):
puts(red('Bitbucket repository creation aborted.'))
return 1
if username and password and organization and repository:
env.box_auth = '%s:"%s"' % (username, password)
env.box_repo = '%s/%s' % (organization, repository)
with hide('running'):
run_local(
'curl'
' -X POST -v -u %(box_auth)s'
' -H "content-type: application/json"'
' https://api.bitbucket.org/2.0/repositories/%(box_repo)s'
' -d \'{"scm": "git", "is_private": true,'
' "forking_policy": "no_public_forks"}\'')
with hide('everything'):
with settings(warn_only=True):
run_local('git remote rm origin')
run_local('git remote add origin [email protected]:%(box_repo)s.git')
run_local('git push -u origin master')
def load_db(filename=None):
"""Loads a dump into the database"""
env.box_dump_filename = filename
if not filename:
abort(red('Dump missing. "fab local.load_db:filename"', bold=True))
if not os.path.exists(filename):
abort(red('"%(box_dump_filename)s" does not exist.' % env, bold=True))
run_local('dropdb --if-exists %(box_database_local)s')
run_local('createdb %(box_database_local)s'
' --encoding=UTF8 --template=template0')
run_local('psql %(box_database_local)s < %(box_dump_filename)s')
def init_bitbucket():
username = default_env("BITBUCKET_USERNAME")
password = default_env("BITBUCKET_PASSWORD") # Should probably not be used
organization = default_env("BITBUCKET_ORGANIZATION")
if not username or not organization:
puts("Consider adding default values for BITBUCKET_USERNAME"
" and BITBUCKET_ORGANIZATION to ~/.box.env")
username = prompt("Username", default=username)
if not password:
password = getpass.getpass("Password ")
organization = prompt("Organization", default=organization)
repository = prompt("Repository", default=env.box_domain)
if not confirm("Initialize repository at https://bitbucket.org/%s/%s?" %
(organization, repository)):
puts(red("Bitbucket repository creation aborted."))
return 1
if username and password and organization and repository:
env.box_auth = '%s:"%s"' % (username, password)
env.box_repo = "%s/%s" % (organization, repository)
with hide("running"):
run_local(
"curl"
" -X POST -v -u %(box_auth)s"
' -H "content-type: application/json"'
" https://api.bitbucket.org/2.0/repositories/%(box_repo)s"
' -d \'{"scm": "git", "is_private": true,'
' "forking_policy": "no_public_forks"}\'')
with hide("everything"):
with settings(warn_only=True):
run_local("git remote rm origin")
run_local("git remote add origin [email protected]:%(box_repo)s.git")
run_local("git push -u origin master")
def pull_database():
"""Pulls the database contents from the server, dropping the local
database first (if it exists)"""
if not confirm('Completely replace the local database'
' "%(box_database_local)s" (if it exists)?'):
return
env.box_remote_db = remote_env('DATABASE_URL')
if not env.box_remote_db:
abort(red('Unable to determine the remote DATABASE_URL', bold=True))
run_local('dropdb --if-exists %(box_database_local)s')
run_local('createdb %(box_database_local)s'
' --encoding=UTF8 --template=template0')
run_local('ssh %(host_string)s pg_dump -Ox "%(box_remote_db)s"'
' | psql %(box_database_local)s')
def direct():
"""Deploys code directly, most useful when Bitbucket is down"""
execute("check.deploy")
step("\nCompiling static sources...")
run_local("yarn run prod")
step("\nPushing changes...")
run_local("git push %(box_remote)s %(box_branch)s:refs/heads/DIRECTDEPLOY")
step("\nDeploying new code on server...")
with cd("%(box_domain)s"):
run("git merge --ff-only DIRECTDEPLOY")
_do_deploy()
run_local("git push %(box_remote)s :refs/heads/DIRECTDEPLOY")
step("\nPLEASE do not forget to push to the source repository anyway!")
def deploy(*args):
"""Deploys frontend and backend code to the server if the checking step
did not report any problems"""
step("\nChecking whether we are up to date...")
run_local("git push --dry-run origin %(box_branch)s")
execute("check.deploy")
step("\nCompiling static sources...")
run_local("yarn run prod")
step("\nPushing changes...")
run_local("git push --all origin %(box_branch)s")
step("\nDeploying new code on server...")
with cd("%(box_domain)s"):
run("git fetch")
run("git merge --ff-only origin/%(box_branch)s")
_do_deploy(args)
def update_requirements():
"""Update requirements.txt"""
run_local('rm -rf venv requirements.txt')
execute('local.create_virtualenv')
def backend_tools():
run_local('venv/bin/pip install -r requirements.txt')
run_local('find . -name "*.pyc" -delete')
run_local('venv/bin/python manage.py migrate')
def frontend_tools():
"""Installs frontend tools. Knows how to handle npm/bower and bundler"""
run_local('yarn')
def reset_passwords():
run_local('venv/bin/python manage.py shell -c "'
'from django.contrib.auth import get_user_model;'
'c=get_user_model();u=c();u.set_password(\'password\');'
'c.objects.update(password=u.password)"')
puts(green('All users now have a password of "password".'))
def test():
step("\nRunning the test suite...")
for cmd in env["box_test"]:
run_local(cmd)
