def load_db(filename=None): """Loads a dump into the database""" env.box_dump_filename = filename if not filename: abort(red('Dump missing. "fab server.load_db:filename"', bold=True)) if not os.path.exists(filename): abort(red('"%(box_dump_filename)s" does not exist.' % env, bold=True)) if not confirm( "Completely replace the remote database" ' "%(box_database)s" (if it exists)?', default=False, ): return env.box_remote_db = remote_env("DATABASE_URL") if not env.box_remote_db: abort(red("Unable to determine the remote DATABASE_URL", bold=True)) run('psql -c "DROP DATABASE IF EXISTS %(box_database)s"') run( "createdb %(box_database)s --encoding=UTF8 --template=template0" " --owner=%(box_database)s" ) run_local( "cat %(box_dump_filename)s |" " ssh %(host_string)s psql %(box_remote_db)s" ) run('psql %(box_database)s -c "REASSIGN OWNED BY admin ' ' TO %(box_database)s"')
def fetch_remote(): step("Updating git remote...") env.box_idx = ("" if len(env.hosts) < 2 else "-%d" % (env.hosts.index(env.host_string) + 1)) with settings(warn_only=True): run_local("git fetch %(box_remote)s%(box_idx)s")
def deploy(): """Checks whether everything is ready for deployment""" step('Checking whether we are on the expected branch...') with settings(warn_only=True), hide('everything'): branch = run_local('git symbolic-ref -q --short HEAD', capture=True) if not branch: abort(red('No branch checked out, cannot continue.', bold=True)) if branch != env.box_branch: puts( red('Warning: The currently checked out branch is \'%s\', but' ' the environment \'%s\' runs on \'%s\'.' % (branch, env.box_environment, env.box_branch))) if not confirm('Continue deployment?', default=False): abort('Aborting.') step('\nChecking whether we are up to date...') run_local('git push --dry-run origin %(box_branch)s') execute('check.check') execute('check.test') with cd('%(box_domain)s'): step('\nChecking for uncommitted changes on the server...') result = run('git status --porcelain') if result: abort(red('Uncommitted changes detected, aborting deployment.'))
def dev(host="127.0.0.1", port=8000): """Runs the development server, SCSS watcher and backend services if they are not running already""" if host == "net": host = own_ip() puts( green("Starting dev server on http://%s:%s/" % (host, port), bold=True)) with tempfile.NamedTemporaryFile() as f: # https://gist.github.com/jiaaro/b2e1b7c705022c2cf56888152a999f65 f.write("""\ trap "exit" INT TERM trap "kill 0" EXIT PYTHONWARNINGS=always venv/bin/python manage.py runserver 0.0.0.0:%(port)s & HOST=%(host)s yarn run dev & for job in $(jobs -p); do wait $job; done """ % { "port": port, "host": host }) f.flush() run_local("bash %s" % f.name)
def fetch_remote(): step('Updating git remote...') env.box_idx = '' if len( env.hosts) < 2 else '-%d' % (env.hosts.index(env.host_string) + 1) with settings(warn_only=True): run_local('git fetch %(box_remote)s%(box_idx)s')
def pull(): # local.backend_tools without migrate step run_local('venv/bin/pip install -r requirements.txt') run_local('find . -name "*.pyc" -delete') execute('local.pull_database') execute('local.update') execute('local.reset_passwords')
def add_remote(): """ Add a server repository as git remote """ env.box_idx = '' if len( env.hosts) < 2 else '-%d' % (env.hosts.index(env.host_string) + 1) with settings(warn_only=True): run_local('git remote add -f %(box_remote)s%(box_idx)s' ' %(host_string)s:%(box_domain)s/')
def dump_db(): """Dumps the database into the tmp/ folder""" env.box_datetime = datetime.now().strftime('%Y-%m-%d-%s') env.box_dump_filename = os.path.join( os.getcwd(), 'tmp', '%(box_database_local)s-local-%(box_datetime)s.sql' % env, ) run_local('pg_dump -Ox %(box_database_local)s > %(box_dump_filename)s') puts(green('\nWrote a dump to %(box_dump_filename)s' % env))
def create_virtualenv(): """Creates the virtualenv and installs all Python requirements""" run_local('python3 -m venv venv') run_local('venv/bin/pip install -U wheel pip') if os.path.exists('requirements.txt'): run_local('venv/bin/pip install -r requirements.txt') else: run_local('venv/bin/pip install -U -r requirements-to-freeze.txt') run_local('(printf "# AUTOGENERATED, DO NOT EDIT\n\n";' 'venv/bin/pip freeze -l' # Until Ubuntu gets its act together: ' | grep -vE "(^pkg-resources)"' ') > requirements.txt')
def create_database(): """Creates and migrates a Postgres database""" if not confirm('Completely replace the local database' ' "%(box_database_local)s" (if it exists)?'): return run_local('dropdb --if-exists %(box_database_local)s') run_local('createdb %(box_database_local)s' ' --encoding=UTF8 --template=template0') with settings(warn_only=True): run_local('venv/bin/python manage.py makemigrations elephantblog') run_local('venv/bin/python manage.py makemigrations page') run_local('venv/bin/python manage.py migrate')
def deploy(): """Checks whether everything is ready for deployment""" step("Checking whether we are on the expected branch...") with settings(warn_only=True), hide("everything"): branch = run_local("git symbolic-ref -q --short HEAD", capture=True) if not branch: abort(red("No branch checked out, cannot continue.", bold=True)) if branch != env.box_branch: puts( red("Warning: The currently checked out branch is '%s', but" " the environment '%s' runs on '%s'." % (branch, env.box_environment, env.box_branch))) if not confirm("Continue deployment?", default=False): abort("Aborting.") execute("check.check") execute("check.test") with cd("%(box_domain)s"): step("\nChecking for uncommitted changes on the server...") result = run("git status --porcelain") if result: abort(red("Uncommitted changes detected, aborting deployment."))
def dump_db(): """Dumps the database into the tmp/ folder""" env.box_datetime = datetime.now().strftime('%Y-%m-%d-%s') env.box_dump_filename = os.path.join( os.getcwd(), 'tmp', '%(box_database)s-%(box_environment)s-%(box_datetime)s.sql' % env, ) env.box_remote_db = remote_env('DATABASE_URL') if not env.box_remote_db: abort(red('Unable to determine the remote DATABASE_URL', bold=True)) run_local('ssh %(host_string)s pg_dump -Ox %(box_remote_db)s' ' > %(box_dump_filename)s') puts(green('\nWrote a dump to %(box_dump_filename)s' % env))
def dump_db(): """Dumps the database into the tmp/ folder""" env.box_datetime = datetime.now().strftime("%Y-%m-%d-%s") env.box_dump_filename = os.path.join( os.getcwd(), "tmp", "%(box_database)s-%(box_environment)s-%(box_datetime)s.sql" % env, ) env.box_remote_db = remote_env("DATABASE_URL") if not env.box_remote_db: abort(red("Unable to determine the remote DATABASE_URL", bold=True)) run_local( "ssh %(host_string)s pg_dump -Ox %(box_remote_db)s" " > %(box_dump_filename)s" ) puts(green("\nWrote a dump to %(box_dump_filename)s" % env))
def mm(): """Wrapper around the ``makemessages`` management command which excludes dependencies (virtualenv, bower components, node modules)""" run_local('venv/bin/python manage.py makemessages -a' ' -i app/cms' ' -i bower_components' ' -i node_modules' ' -i venv') """Also statici18n ``makemessages`` command will be executed""" run_local('venv/bin/python manage.py makemessages -d djangojs -a' ' -e jsx,js' ' -i app/static/jsi18n' ' -i app/cms' ' -i app/templates/elephantblog' ' -i bower_components' ' -i node_modules' ' -i venv')
def dev(host='127.0.0.1', port=8000): """Runs the development server, SCSS watcher and backend services if they are not running already""" if host == 'net': host = own_ip() puts( green('Starting dev server on http://%s:%s/' % (host, port), bold=True)) jobs = [ lambda: run_local( 'venv/bin/python -Wonce manage.py runserver 0.0.0.0:%s' % (port, ), ), lambda: run_local('HOST=%s yarn run dev' % host), ] jobs = [Process(target=j) for j in jobs] [j.start() for j in jobs] [j.join() for j in jobs]
def deploy(*args): """Deploys frontend and backend code to the server if the checking step did not report any problems""" execute('check.deploy') step('\nCompiling static sources...') run_local('yarn run prod') step('\nPushing changes...') run_local('git push origin %(box_branch)s') step('\nDeploying new code on server...') with cd('%(box_domain)s'): run('git fetch') run('git reset --hard origin/%(box_branch)s') run('find . -name "*.pyc" -delete') run('venv/bin/pip install -r requirements.txt') run('venv/bin/python manage.py migrate --noinput') step('\nUploading static files...') rsync_project( local_dir='static/', remote_dir='%(box_domain)s/static/' % env, delete=('clear' in args), ) step('\nCollecting static files...') with cd('%(box_domain)s'): run('venv/bin/python manage.py collectstatic --noinput') step('\nRunning system checks on server...') with cd('%(box_domain)s'): run('venv/bin/python manage.py check --deploy') step('\nRestarting server process...') for line in env['box_restart']: run(line) execute('git.fetch_remote')
def mm(language=None): """Wrapper around the ``makemessages`` management command which excludes dependencies (virtualenv, bower components, node modules)""" run_local(" ".join([ "venv/bin/python manage.py makemessages", ("-l %s" % language) if language else "-a", "-i app/cms", "-i bower_components", "-i node_modules", "-i venv", ])) """Also statici18n ``makemessages`` command will be executed""" run_local(" ".join([ "venv/bin/python manage.py makemessages -d djangojs", ("-l %s" % language) if language else "-a", "-e jsx,js", "-i app/static/jsi18n", "-i app/cms", "-i app/templates/elephantblog", "-i bower_components", "-i node_modules", "-i venv", ]))
def clone_repository(): puts(green('We need the repository to initialize the server.')) with hide('running'): output = run_local('git config remote.origin.url', capture=True) repo = prompt('Repository', default=output) if not repo: puts(red('Cannot continue without a repository.')) return 1 env.box_repository_url = repo run('git clone -b %(box_branch)s %(box_repository_url)s %(box_domain)s') execute('git.add_remote')
def clone_repository(): puts(green("We need the repository to initialize the server.")) with hide("running"): output = run_local("git config remote.origin.url", capture=True) repo = prompt("Repository", default=output) if not repo: puts(red("Cannot continue without a repository.")) return 1 env.box_repository_url = repo run("git clone -b %(box_branch)s %(box_repository_url)s %(box_domain)s") execute("git.add_remote")
def init_bitbucket(): username = default_env('BITBUCKET_USERNAME') password = default_env('BITBUCKET_PASSWORD') # Should probably not be used organization = default_env('BITBUCKET_ORGANIZATION') if not username or not organization: puts('Consider adding default values for BITBUCKET_USERNAME' ' and BITBUCKET_ORGANIZATION to ~/.box.env') username = prompt('Username', default=username) if not password: password = getpass.getpass('Password ') organization = prompt('Organization', default=organization) repository = prompt('Repository', default=env.box_repository) if not confirm('Initialize repository at https://bitbucket.org/%s/%s?' % (organization, repository)): puts(red('Bitbucket repository creation aborted.')) return 1 if username and password and organization and repository: env.box_auth = '%s:"%s"' % (username, password) env.box_repo = '%s/%s' % (organization, repository) with hide('running'): run_local( 'curl' ' -X POST -v -u %(box_auth)s' ' -H "content-type: application/json"' ' https://api.bitbucket.org/2.0/repositories/%(box_repo)s' ' -d \'{"scm": "git", "is_private": true,' ' "forking_policy": "no_public_forks"}\'') with hide('everything'): with settings(warn_only=True): run_local('git remote rm origin') run_local('git remote add origin [email protected]:%(box_repo)s.git') run_local('git push -u origin master')
def load_db(filename=None): """Loads a dump into the database""" env.box_dump_filename = filename if not filename: abort(red('Dump missing. "fab local.load_db:filename"', bold=True)) if not os.path.exists(filename): abort(red('"%(box_dump_filename)s" does not exist.' % env, bold=True)) run_local('dropdb --if-exists %(box_database_local)s') run_local('createdb %(box_database_local)s' ' --encoding=UTF8 --template=template0') run_local('psql %(box_database_local)s < %(box_dump_filename)s')
def init_bitbucket(): username = default_env("BITBUCKET_USERNAME") password = default_env("BITBUCKET_PASSWORD") # Should probably not be used organization = default_env("BITBUCKET_ORGANIZATION") if not username or not organization: puts("Consider adding default values for BITBUCKET_USERNAME" " and BITBUCKET_ORGANIZATION to ~/.box.env") username = prompt("Username", default=username) if not password: password = getpass.getpass("Password ") organization = prompt("Organization", default=organization) repository = prompt("Repository", default=env.box_domain) if not confirm("Initialize repository at https://bitbucket.org/%s/%s?" % (organization, repository)): puts(red("Bitbucket repository creation aborted.")) return 1 if username and password and organization and repository: env.box_auth = '%s:"%s"' % (username, password) env.box_repo = "%s/%s" % (organization, repository) with hide("running"): run_local( "curl" " -X POST -v -u %(box_auth)s" ' -H "content-type: application/json"' " https://api.bitbucket.org/2.0/repositories/%(box_repo)s" ' -d \'{"scm": "git", "is_private": true,' ' "forking_policy": "no_public_forks"}\'') with hide("everything"): with settings(warn_only=True): run_local("git remote rm origin") run_local("git remote add origin [email protected]:%(box_repo)s.git") run_local("git push -u origin master")
def pull_database(): """Pulls the database contents from the server, dropping the local database first (if it exists)""" if not confirm('Completely replace the local database' ' "%(box_database_local)s" (if it exists)?'): return env.box_remote_db = remote_env('DATABASE_URL') if not env.box_remote_db: abort(red('Unable to determine the remote DATABASE_URL', bold=True)) run_local('dropdb --if-exists %(box_database_local)s') run_local('createdb %(box_database_local)s' ' --encoding=UTF8 --template=template0') run_local('ssh %(host_string)s pg_dump -Ox "%(box_remote_db)s"' ' | psql %(box_database_local)s')
def direct(): """Deploys code directly, most useful when Bitbucket is down""" execute("check.deploy") step("\nCompiling static sources...") run_local("yarn run prod") step("\nPushing changes...") run_local("git push %(box_remote)s %(box_branch)s:refs/heads/DIRECTDEPLOY") step("\nDeploying new code on server...") with cd("%(box_domain)s"): run("git merge --ff-only DIRECTDEPLOY") _do_deploy() run_local("git push %(box_remote)s :refs/heads/DIRECTDEPLOY") step("\nPLEASE do not forget to push to the source repository anyway!")
def deploy(*args): """Deploys frontend and backend code to the server if the checking step did not report any problems""" step("\nChecking whether we are up to date...") run_local("git push --dry-run origin %(box_branch)s") execute("check.deploy") step("\nCompiling static sources...") run_local("yarn run prod") step("\nPushing changes...") run_local("git push --all origin %(box_branch)s") step("\nDeploying new code on server...") with cd("%(box_domain)s"): run("git fetch") run("git merge --ff-only origin/%(box_branch)s") _do_deploy(args)
def update_requirements(): """Update requirements.txt""" run_local('rm -rf venv requirements.txt') execute('local.create_virtualenv')
def backend_tools(): run_local('venv/bin/pip install -r requirements.txt') run_local('find . -name "*.pyc" -delete') run_local('venv/bin/python manage.py migrate')
def frontend_tools(): """Installs frontend tools. Knows how to handle npm/bower and bundler""" run_local('yarn')
def reset_passwords(): run_local('venv/bin/python manage.py shell -c "' 'from django.contrib.auth import get_user_model;' 'c=get_user_model();u=c();u.set_password(\'password\');' 'c.objects.update(password=u.password)"') puts(green('All users now have a password of "password".'))
def test(): step("\nRunning the test suite...") for cmd in env["box_test"]: run_local(cmd)