def remove_host():
if not confirm(
'Really remove the host "%(box_domain)s" and all associated data?',
default=False,
):
return
run("sudo nine-manage-vhosts virtual-host remove %(box_domain)s")
for line in env["box_disable_process"]:
run(line)
with cd(env.box_domain):
env.box_datetime = datetime.now().strftime("%Y-%m-%d-%s")
run(
"pg_dump -Ox %(box_database)s"
" > %(box_database)s-%(box_environment)s-%(box_datetime)s.sql"
)
run("dropdb %(box_database)s")
run("dropuser %(box_database)s")
puts(
red(
"The folder ~/%(box_domain)s on the server has not been removed. The"
" project folder also contains a fresh database dump." % env
)
)
def fmt(ctx):
fl.fmt(ctx)
fl.run(
ctx,
"yarn run prettier --write --no-semi --trailing-comma es5"
' "absences/**/*.*" "planning/**/*.*" "timer/**/*.*"',
)
def deploy_code(ctx):
check(ctx)
fl.run(ctx, "git push origin main")
with Connection(config.host) as conn:
_do_deploy(conn, "www/workbench/", rsync=False)
_restart_all(conn)
fl.fetch(ctx)
def deploy(ctx):
fl._check_branch(ctx)
check(ctx)
fl.run(ctx, "git push origin main")
fl.run(ctx, "NODE_ENV=production yarn run webpack -p --bail")
with Connection(config.host) as conn:
_do_deploy(conn, "www/workbench/", rsync=True)
_restart_all(conn)
fl.fetch(ctx)
def add_alias(alias):
env.box_nmv_alias = alias
run(
"sudo nine-manage-vhosts alias create %(box_nmv_alias)s"
" --virtual-host=%(box_domain)s"
)
run(
"sudo nine-manage-vhosts alias create www.%(box_nmv_alias)s"
" --virtual-host=%(box_domain)s"
)
def remove_alias(alias):
env.box_nmv_alias = alias
run(
"sudo nine-manage-vhosts alias remove %(box_nmv_alias)s"
" --virtual-host=%(box_domain)s"
)
run(
"sudo nine-manage-vhosts alias remove www.%(box_nmv_alias)s"
" --virtual-host=%(box_domain)s"
)
def clone_repository():
puts(green('We need the repository to initialize the server.'))
with hide('running'):
output = run_local('git config remote.origin.url', capture=True)
repo = prompt('Repository', default=output)
if not repo:
puts(red('Cannot continue without a repository.'))
return 1
env.box_repository_url = repo
run('git clone -b %(box_branch)s %(box_repository_url)s %(box_domain)s')
execute('git.add_remote')
def clone_repository():
puts(green("We need the repository to initialize the server."))
with hide("running"):
output = run_local("git config remote.origin.url", capture=True)
repo = prompt("Repository", default=output)
if not repo:
puts(red("Cannot continue without a repository."))
return 1
env.box_repository_url = repo
run("git clone -b %(box_branch)s %(box_repository_url)s %(box_domain)s")
execute("git.add_remote")
def copy_data_from(environment=None):
"""
Copy the database from one environment to another. Usually from production
to stage.
Usage: ``fab s server.copy_data_from:production``.
:param environment: the source environment
"""
if env.get("box_hardwired_environment"):
abort(red("Cannot continue with a hardwired environment."))
if environment not in env.box_environments:
abort(red("Invalid environment %s." % environment))
source = env.box_environments[environment]
target = env.box_environments[env.get("box_environment")]
if source == target:
abort(
red(
"Source environment %s must not equal target environment %s."
% (environment, env.get("box_environment"))
)
)
if source["servers"][0] != target["servers"][0]:
abort(red("The environments have to be on the same server, sorry!"))
puts("Copying data from {0} to {1}".format(source["remote"], target["remote"]))
if not confirm(
"Completely replace the remote database" ' "%(box_database)s" (if it exists)?',
default=False,
):
return
for key, value in source.items():
env["source_%s" % key] = value
with settings(warn_only=True):
run("dropdb %(box_database)s")
run(
"createdb %(box_database)s --encoding=UTF8 --template=template0"
" --owner=%(box_database)s"
)
run("pg_dump -Ox %(source_database)s | psql %(box_database)s")
run('psql %(box_database)s -c "REASSIGN OWNED BY admin ' ' TO %(box_database)s"')
with cd(env.box_domain):
run("cp -aln ~/%(source_domain)s/media/* media/")
execute("server.restart")
def direct():
"""Deploys code directly, most useful when Bitbucket is down"""
execute("check.deploy")
step("\nCompiling static sources...")
run_local("yarn run prod")
step("\nPushing changes...")
run_local("git push %(box_remote)s %(box_branch)s:refs/heads/DIRECTDEPLOY")
step("\nDeploying new code on server...")
with cd("%(box_domain)s"):
run("git merge --ff-only DIRECTDEPLOY")
_do_deploy()
run_local("git push %(box_remote)s :refs/heads/DIRECTDEPLOY")
step("\nPLEASE do not forget to push to the source repository anyway!")
def deploy(*args):
"""Deploys frontend and backend code to the server if the checking step
did not report any problems"""
step("\nChecking whether we are up to date...")
run_local("git push --dry-run origin %(box_branch)s")
execute("check.deploy")
step("\nCompiling static sources...")
run_local("yarn run prod")
step("\nPushing changes...")
run_local("git push --all origin %(box_branch)s")
step("\nDeploying new code on server...")
with cd("%(box_domain)s"):
run("git fetch")
run("git merge --ff-only origin/%(box_branch)s")
_do_deploy(args)
def load_db(filename=None):
"""Loads a dump into the database"""
env.box_dump_filename = filename
if not filename:
abort(red('Dump missing. "fab server.load_db:filename"', bold=True))
if not os.path.exists(filename):
abort(red('"%(box_dump_filename)s" does not exist.' % env, bold=True))
if not confirm(
"Completely replace the remote database" ' "%(box_database)s" (if it exists)?',
default=False,
):
return
env.box_remote_db = remote_env("DATABASE_URL")
if not env.box_remote_db:
abort(red("Unable to determine the remote DATABASE_URL", bold=True))
run('psql -c "DROP DATABASE IF EXISTS %(box_database)s"')
run(
"createdb %(box_database)s --encoding=UTF8 --template=template0"
" --owner=%(box_database)s"
)
run_local(
"cat %(box_dump_filename)s |" " ssh %(host_string)s psql %(box_remote_db)s"
)
run('psql %(box_database)s -c "REASSIGN OWNED BY admin ' ' TO %(box_database)s"')
def create_database_and_dotenv():
env.box_sentry_dsn = prompt("Sentry DSN")
env.box_oauth2_client_id = prompt("Google OAuth2 Client ID")
env.box_oauth2_client_secret = prompt("Google OAuth2 Client Secret")
env.box_database_pw = get_random_string(
20, chars="abcdefghijklmopqrstuvwx01234567890"
)
env.box_secret_key = get_random_string(50)
run(
'psql -c "CREATE ROLE %(box_database)s WITH'
" ENCRYPTED PASSWORD '%(box_database_pw)s'"
' LOGIN NOCREATEDB NOCREATEROLE NOSUPERUSER"'
)
run('psql -c "GRANT %(box_database)s TO admin"')
run(
'psql -c "CREATE DATABASE %(box_database)s WITH'
" OWNER %(box_database)s"
" TEMPLATE template0"
" ENCODING 'UTF8'\""
)
with cd("%(box_domain)s"):
put(
StringIO(
"""\
DEBUG=False
DATABASE_URL=postgres://%(box_database)s:%(box_database_pw)s\
@localhost:5432/%(box_database)s
CACHE_URL=hiredis://localhost:6379/1/?key_prefix=%(box_database)s
SECRET_KEY=%(box_secret_key)s
SENTRY_DSN=%(box_sentry_dsn)s
ALLOWED_HOSTS=['.%(box_domain)s', '.%(host_string_host)s']
GOOGLE_CLIENT_ID=%(box_oauth2_client_id)s
GOOGLE_CLIENT_SECRET=%(box_oauth2_client_secret)s
# LIVE=True
# CANONICAL_DOMAIN=%(box_domain)s
# CANONICAL_DOMAIN_SECURE=True
"""
% dict(env, host_string_host=env.host_string.split("@")[-1])
),
".env",
)
run("venv/bin/python manage.py migrate --noinput")
def nginx_vhost_and_supervisor():
run('sudo nine-manage-vhosts virtual-host create %(box_domain)s'
' --template=feinheit --webroot=/home/www-data/%(box_domain)s/htdocs')
with cd('%(box_domain)s'):
run('mkdir -p media tmp')
for line in env['box_enable_process']:
run(line)
def pull_db(ctx, installation="fh"):
fl.run(ctx, "dropdb --if-exists workbench", warn=True)
fl.run(ctx, "createdb workbench")
with Connection(config.host) as conn:
e = fl._srv_env(conn, f"www/workbench/.env/{installation}")
srv_dsn = e("DATABASE_URL")
fl.run(
ctx,
f'ssh -C {config.host} "pg_dump -Ox {srv_dsn}" | psql workbench',
)
def nginx_vhost_and_supervisor():
run(
"sudo nine-manage-vhosts virtual-host create %(box_domain)s"
" --template=%(box_vhost_template)s"
" --webroot=/home/www-data/%(box_domain)s/htdocs"
)
with cd("%(box_domain)s"):
run("mkdir -p media tmp")
for line in env["box_enable_process"]:
run(line)
def create_database_and_dotenv():
env.box_sentry_dsn = prompt('Sentry DSN')
env.box_oauth2_client_id = prompt('Google OAuth2 Client ID')
env.box_oauth2_client_secret = prompt('Google OAuth2 Client Secret')
env.box_database_pw = get_random_string(
20, chars='abcdefghijklmopqrstuvwx01234567890')
env.box_secret_key = get_random_string(50)
run('psql -c "CREATE ROLE %(box_database)s WITH'
' ENCRYPTED PASSWORD \'%(box_database_pw)s\''
' LOGIN NOCREATEDB NOCREATEROLE NOSUPERUSER"')
run('psql -c "GRANT %(box_database)s TO admin"')
run('psql -c "CREATE DATABASE %(box_database)s WITH'
' OWNER %(box_database)s'
' TEMPLATE template0'
' ENCODING \'UTF8\'"')
with cd('%(box_domain)s'):
put(
StringIO('''\
DATABASE_URL=postgres://%(box_database)s:%(box_database_pw)s\
@localhost:5432/%(box_database)s
CACHE_URL=hiredis://localhost:6379/1/?key_prefix=%(box_database)s
SECRET_KEY=%(box_secret_key)s
SENTRY_DSN=%(box_sentry_dsn)s
ALLOWED_HOSTS=['.%(box_domain)s', '.%(host_string_host)s']
GOOGLE_CLIENT_ID=%(box_oauth2_client_id)s
GOOGLE_CLIENT_SECRET=%(box_oauth2_client_secret)s
# LIVE=True
# CANONICAL_DOMAIN=%(box_domain)s
# CANONICAL_DOMAIN_SECURE=True
''' % dict(env, host_string_host=env.host_string.split('@')[-1])), '.env')
run('venv/bin/python manage.py migrate --noinput')
def check(ctx):
fl.check(ctx)
fl.run(
ctx,
"yarn run prettier --list-different --no-semi --trailing-comma es5"
' "absences/**/*.*" "planning/**/*.*" "timer/**/*.*"',
)
fl.run(
ctx,
'yarn run eslint "absences/**/*.js" "planning/**/*.js" "timer/**/*.js"',
)
fl.run(
ctx,
"pipx run interrogate"
" -e node_modules -e venv -v -f 99 --whitelist-regex 'test_.*'",
)
def _restart_all(conn):
for wb in config.installations:
fl.run(conn, "systemctl --user restart workbench@{}".format(wb), echo=True)
def _do_deploy(conn, folder, rsync):
with conn.cd(folder):
fl.run(conn, "git checkout main")
fl.run(conn, "git fetch origin")
fl.run(conn, "git merge --ff-only origin/main")
fl.run(conn, 'find . -name "*.pyc" -delete')
fl.run(conn, "venv/bin/pip install -U 'pip!=20.3.2' wheel setuptools")
fl.run(conn, "venv/bin/pip install -r requirements.txt")
for wb in config.installations:
fl.run(conn, "DOTENV=.env/{} venv/bin/python manage.py migrate".format(wb))
if rsync:
conn.local(
"rsync -avz --delete static/ {}:{}static".format(config.host, folder)
)
with conn.cd(folder):
fl.run(
conn,
"DOTENV=.env/{} venv/bin/python manage.py collectstatic --noinput".format(
config.installations[0]
),
)
def mm(ctx):
fl.run(
ctx,
"venv/bin/python extract_gettext.py > conf/strings.js",
)
fl.mm(ctx)
def create_virtualenv():
with cd("%(box_domain)s"):
run("rm -rf venv")
run("python3 -m venv venv")
run("venv/bin/pip install -U pip wheel setuptools")
run("venv/bin/pip install -r requirements.txt")
def restart():
"""Restart the server process"""
for line in env["box_restart"]:
run(line)
def deploy(*args):
"""Deploys frontend and backend code to the server if the checking step
did not report any problems"""
execute('check.deploy')
step('\nCompiling static sources...')
run_local('yarn run prod')
step('\nPushing changes...')
run_local('git push origin %(box_branch)s')
step('\nDeploying new code on server...')
with cd('%(box_domain)s'):
run('git fetch')
run('git reset --hard origin/%(box_branch)s')
run('find . -name "*.pyc" -delete')
run('venv/bin/pip install -r requirements.txt')
run('venv/bin/python manage.py migrate --noinput')
step('\nUploading static files...')
rsync_project(
local_dir='static/',
remote_dir='%(box_domain)s/static/' % env,
delete=('clear' in args),
)
step('\nCollecting static files...')
with cd('%(box_domain)s'):
run('venv/bin/python manage.py collectstatic --noinput')
step('\nRunning system checks on server...')
with cd('%(box_domain)s'):
run('venv/bin/python manage.py check --deploy')
step('\nRestarting server process...')
for line in env['box_restart']:
run(line)
execute('git.fetch_remote')
def _restart_all(conn):
for wb in fl.config.installations:
fl.run(conn, f"systemctl --user restart workbench@{wb}", echo=True)
def create_virtualenv():
with cd('%(box_domain)s'):
run('rm -rf venv')
run('python3 -m venv venv')
run('venv/bin/pip install -U pip wheel')
run('venv/bin/pip install -r requirements.txt')
def dbshell():
# env.box_remote_db = remote_env('DATABASE_URL')
# ssh SERVER -o RequestTTY=yes\
# 'psql $(grep -E "^DATABASE_URL=" %(box_domain)s/.env | cut -f2 -d=)'
run("psql %(box_database)s")
def ssl():
run("sudo nine-manage-vhosts certificate create" " --virtual-host=%(box_domain)s")
run(
"sudo nine-manage-vhosts virtual-host update %(box_domain)s"
" --template=%(box_vhost_ssl_template)s"
)
def _do_deploy(args=()):
with cd("%(box_domain)s"):
run('find . -name "*.pyc" -delete')
run("venv/bin/pip install -U pip wheel setuptools")
run("venv/bin/pip install -r requirements.txt")
run("venv/bin/python manage.py migrate --noinput")
step("\nUploading static files...")
rsync_project(
local_dir="static/",
remote_dir="%(box_domain)s/static/" % env,
delete=("clear" in args),
)
step("\nCollecting static files...")
with cd("%(box_domain)s"):
run("venv/bin/python manage.py collectstatic --noinput")
step("\nRunning system checks on server...")
with cd("%(box_domain)s"):
run("venv/bin/python manage.py check --deploy")
step("\nRestarting server process...")
execute("server.restart")
execute("git.fetch_remote")
def copy_data_from(environment=None):
"""
Copy the database from one environment to another. Usually from production
to stage.
Usage: ``fab s server.copy_data_from:production``.
:param environment: the source environment
"""
if env.get('box_hardwired_environment'):
abort(red('Cannot continue with a hardwired environment.'))
if environment not in env.box_environments:
abort(red('Invalid environment %s.' % environment))
source = env.box_environments[environment]
target = env.box_environments[env.get('box_environment')]
if source == target:
abort(
red('Source environment %s must not equal target environment %s.' %
(environment, env.get('box_environment'))))
if source['servers'][0] != target['servers'][0]:
abort(red('The environments have to be on the same server, sorry!'))
puts('Copying data from {0} to {1}'.format(source['remote'],
target['remote']))
if not confirm(
'Completely replace the remote database'
' "%(box_database)s" (if it exists)?',
default=False):
return
for key, value in source.items():
env['source_%s' % key] = value
with settings(warn_only=True):
run('dropdb %(box_database)s')
run('createdb %(box_database)s --encoding=UTF8 --template=template0'
' --owner=%(box_database)s')
run('pg_dump -Ox %(source_database)s | psql %(box_database)s')
run('psql %(box_database)s -c "REASSIGN OWNED BY admin '
' TO %(box_database)s"')
with cd(env.box_domain):
run('cp -aln ~/%(source_domain)s/media/* media/')
for line in env['box_restart']:
run(line)
