def remove_host(): if not confirm( 'Really remove the host "%(box_domain)s" and all associated data?', default=False, ): return run("sudo nine-manage-vhosts virtual-host remove %(box_domain)s") for line in env["box_disable_process"]: run(line) with cd(env.box_domain): env.box_datetime = datetime.now().strftime("%Y-%m-%d-%s") run( "pg_dump -Ox %(box_database)s" " > %(box_database)s-%(box_environment)s-%(box_datetime)s.sql" ) run("dropdb %(box_database)s") run("dropuser %(box_database)s") puts( red( "The folder ~/%(box_domain)s on the server has not been removed. The" " project folder also contains a fresh database dump." % env ) )
def fmt(ctx): fl.fmt(ctx) fl.run( ctx, "yarn run prettier --write --no-semi --trailing-comma es5" ' "absences/**/*.*" "planning/**/*.*" "timer/**/*.*"', )
def deploy_code(ctx): check(ctx) fl.run(ctx, "git push origin main") with Connection(config.host) as conn: _do_deploy(conn, "www/workbench/", rsync=False) _restart_all(conn) fl.fetch(ctx)
def deploy(ctx): fl._check_branch(ctx) check(ctx) fl.run(ctx, "git push origin main") fl.run(ctx, "NODE_ENV=production yarn run webpack -p --bail") with Connection(config.host) as conn: _do_deploy(conn, "www/workbench/", rsync=True) _restart_all(conn) fl.fetch(ctx)
def add_alias(alias): env.box_nmv_alias = alias run( "sudo nine-manage-vhosts alias create %(box_nmv_alias)s" " --virtual-host=%(box_domain)s" ) run( "sudo nine-manage-vhosts alias create www.%(box_nmv_alias)s" " --virtual-host=%(box_domain)s" )
def remove_alias(alias): env.box_nmv_alias = alias run( "sudo nine-manage-vhosts alias remove %(box_nmv_alias)s" " --virtual-host=%(box_domain)s" ) run( "sudo nine-manage-vhosts alias remove www.%(box_nmv_alias)s" " --virtual-host=%(box_domain)s" )
def clone_repository(): puts(green('We need the repository to initialize the server.')) with hide('running'): output = run_local('git config remote.origin.url', capture=True) repo = prompt('Repository', default=output) if not repo: puts(red('Cannot continue without a repository.')) return 1 env.box_repository_url = repo run('git clone -b %(box_branch)s %(box_repository_url)s %(box_domain)s') execute('git.add_remote')
def clone_repository(): puts(green("We need the repository to initialize the server.")) with hide("running"): output = run_local("git config remote.origin.url", capture=True) repo = prompt("Repository", default=output) if not repo: puts(red("Cannot continue without a repository.")) return 1 env.box_repository_url = repo run("git clone -b %(box_branch)s %(box_repository_url)s %(box_domain)s") execute("git.add_remote")
def copy_data_from(environment=None): """ Copy the database from one environment to another. Usually from production to stage. Usage: ``fab s server.copy_data_from:production``. :param environment: the source environment """ if env.get("box_hardwired_environment"): abort(red("Cannot continue with a hardwired environment.")) if environment not in env.box_environments: abort(red("Invalid environment %s." % environment)) source = env.box_environments[environment] target = env.box_environments[env.get("box_environment")] if source == target: abort( red( "Source environment %s must not equal target environment %s." % (environment, env.get("box_environment")) ) ) if source["servers"][0] != target["servers"][0]: abort(red("The environments have to be on the same server, sorry!")) puts("Copying data from {0} to {1}".format(source["remote"], target["remote"])) if not confirm( "Completely replace the remote database" ' "%(box_database)s" (if it exists)?', default=False, ): return for key, value in source.items(): env["source_%s" % key] = value with settings(warn_only=True): run("dropdb %(box_database)s") run( "createdb %(box_database)s --encoding=UTF8 --template=template0" " --owner=%(box_database)s" ) run("pg_dump -Ox %(source_database)s | psql %(box_database)s") run('psql %(box_database)s -c "REASSIGN OWNED BY admin ' ' TO %(box_database)s"') with cd(env.box_domain): run("cp -aln ~/%(source_domain)s/media/* media/") execute("server.restart")
def direct(): """Deploys code directly, most useful when Bitbucket is down""" execute("check.deploy") step("\nCompiling static sources...") run_local("yarn run prod") step("\nPushing changes...") run_local("git push %(box_remote)s %(box_branch)s:refs/heads/DIRECTDEPLOY") step("\nDeploying new code on server...") with cd("%(box_domain)s"): run("git merge --ff-only DIRECTDEPLOY") _do_deploy() run_local("git push %(box_remote)s :refs/heads/DIRECTDEPLOY") step("\nPLEASE do not forget to push to the source repository anyway!")
def deploy(*args): """Deploys frontend and backend code to the server if the checking step did not report any problems""" step("\nChecking whether we are up to date...") run_local("git push --dry-run origin %(box_branch)s") execute("check.deploy") step("\nCompiling static sources...") run_local("yarn run prod") step("\nPushing changes...") run_local("git push --all origin %(box_branch)s") step("\nDeploying new code on server...") with cd("%(box_domain)s"): run("git fetch") run("git merge --ff-only origin/%(box_branch)s") _do_deploy(args)
def load_db(filename=None): """Loads a dump into the database""" env.box_dump_filename = filename if not filename: abort(red('Dump missing. "fab server.load_db:filename"', bold=True)) if not os.path.exists(filename): abort(red('"%(box_dump_filename)s" does not exist.' % env, bold=True)) if not confirm( "Completely replace the remote database" ' "%(box_database)s" (if it exists)?', default=False, ): return env.box_remote_db = remote_env("DATABASE_URL") if not env.box_remote_db: abort(red("Unable to determine the remote DATABASE_URL", bold=True)) run('psql -c "DROP DATABASE IF EXISTS %(box_database)s"') run( "createdb %(box_database)s --encoding=UTF8 --template=template0" " --owner=%(box_database)s" ) run_local( "cat %(box_dump_filename)s |" " ssh %(host_string)s psql %(box_remote_db)s" ) run('psql %(box_database)s -c "REASSIGN OWNED BY admin ' ' TO %(box_database)s"')
def create_database_and_dotenv(): env.box_sentry_dsn = prompt("Sentry DSN") env.box_oauth2_client_id = prompt("Google OAuth2 Client ID") env.box_oauth2_client_secret = prompt("Google OAuth2 Client Secret") env.box_database_pw = get_random_string( 20, chars="abcdefghijklmopqrstuvwx01234567890" ) env.box_secret_key = get_random_string(50) run( 'psql -c "CREATE ROLE %(box_database)s WITH' " ENCRYPTED PASSWORD '%(box_database_pw)s'" ' LOGIN NOCREATEDB NOCREATEROLE NOSUPERUSER"' ) run('psql -c "GRANT %(box_database)s TO admin"') run( 'psql -c "CREATE DATABASE %(box_database)s WITH' " OWNER %(box_database)s" " TEMPLATE template0" " ENCODING 'UTF8'\"" ) with cd("%(box_domain)s"): put( StringIO( """\ DEBUG=False DATABASE_URL=postgres://%(box_database)s:%(box_database_pw)s\ @localhost:5432/%(box_database)s CACHE_URL=hiredis://localhost:6379/1/?key_prefix=%(box_database)s SECRET_KEY=%(box_secret_key)s SENTRY_DSN=%(box_sentry_dsn)s ALLOWED_HOSTS=['.%(box_domain)s', '.%(host_string_host)s'] GOOGLE_CLIENT_ID=%(box_oauth2_client_id)s GOOGLE_CLIENT_SECRET=%(box_oauth2_client_secret)s # LIVE=True # CANONICAL_DOMAIN=%(box_domain)s # CANONICAL_DOMAIN_SECURE=True """ % dict(env, host_string_host=env.host_string.split("@")[-1]) ), ".env", ) run("venv/bin/python manage.py migrate --noinput")
def nginx_vhost_and_supervisor(): run('sudo nine-manage-vhosts virtual-host create %(box_domain)s' ' --template=feinheit --webroot=/home/www-data/%(box_domain)s/htdocs') with cd('%(box_domain)s'): run('mkdir -p media tmp') for line in env['box_enable_process']: run(line)
def pull_db(ctx, installation="fh"): fl.run(ctx, "dropdb --if-exists workbench", warn=True) fl.run(ctx, "createdb workbench") with Connection(config.host) as conn: e = fl._srv_env(conn, f"www/workbench/.env/{installation}") srv_dsn = e("DATABASE_URL") fl.run( ctx, f'ssh -C {config.host} "pg_dump -Ox {srv_dsn}" | psql workbench', )
def nginx_vhost_and_supervisor(): run( "sudo nine-manage-vhosts virtual-host create %(box_domain)s" " --template=%(box_vhost_template)s" " --webroot=/home/www-data/%(box_domain)s/htdocs" ) with cd("%(box_domain)s"): run("mkdir -p media tmp") for line in env["box_enable_process"]: run(line)
def create_database_and_dotenv(): env.box_sentry_dsn = prompt('Sentry DSN') env.box_oauth2_client_id = prompt('Google OAuth2 Client ID') env.box_oauth2_client_secret = prompt('Google OAuth2 Client Secret') env.box_database_pw = get_random_string( 20, chars='abcdefghijklmopqrstuvwx01234567890') env.box_secret_key = get_random_string(50) run('psql -c "CREATE ROLE %(box_database)s WITH' ' ENCRYPTED PASSWORD \'%(box_database_pw)s\'' ' LOGIN NOCREATEDB NOCREATEROLE NOSUPERUSER"') run('psql -c "GRANT %(box_database)s TO admin"') run('psql -c "CREATE DATABASE %(box_database)s WITH' ' OWNER %(box_database)s' ' TEMPLATE template0' ' ENCODING \'UTF8\'"') with cd('%(box_domain)s'): put( StringIO('''\ DATABASE_URL=postgres://%(box_database)s:%(box_database_pw)s\ @localhost:5432/%(box_database)s CACHE_URL=hiredis://localhost:6379/1/?key_prefix=%(box_database)s SECRET_KEY=%(box_secret_key)s SENTRY_DSN=%(box_sentry_dsn)s ALLOWED_HOSTS=['.%(box_domain)s', '.%(host_string_host)s'] GOOGLE_CLIENT_ID=%(box_oauth2_client_id)s GOOGLE_CLIENT_SECRET=%(box_oauth2_client_secret)s # LIVE=True # CANONICAL_DOMAIN=%(box_domain)s # CANONICAL_DOMAIN_SECURE=True ''' % dict(env, host_string_host=env.host_string.split('@')[-1])), '.env') run('venv/bin/python manage.py migrate --noinput')
def check(ctx): fl.check(ctx) fl.run( ctx, "yarn run prettier --list-different --no-semi --trailing-comma es5" ' "absences/**/*.*" "planning/**/*.*" "timer/**/*.*"', ) fl.run( ctx, 'yarn run eslint "absences/**/*.js" "planning/**/*.js" "timer/**/*.js"', ) fl.run( ctx, "pipx run interrogate" " -e node_modules -e venv -v -f 99 --whitelist-regex 'test_.*'", )
def _restart_all(conn): for wb in config.installations: fl.run(conn, "systemctl --user restart workbench@{}".format(wb), echo=True)
def _do_deploy(conn, folder, rsync): with conn.cd(folder): fl.run(conn, "git checkout main") fl.run(conn, "git fetch origin") fl.run(conn, "git merge --ff-only origin/main") fl.run(conn, 'find . -name "*.pyc" -delete') fl.run(conn, "venv/bin/pip install -U 'pip!=20.3.2' wheel setuptools") fl.run(conn, "venv/bin/pip install -r requirements.txt") for wb in config.installations: fl.run(conn, "DOTENV=.env/{} venv/bin/python manage.py migrate".format(wb)) if rsync: conn.local( "rsync -avz --delete static/ {}:{}static".format(config.host, folder) ) with conn.cd(folder): fl.run( conn, "DOTENV=.env/{} venv/bin/python manage.py collectstatic --noinput".format( config.installations[0] ), )
def mm(ctx): fl.run( ctx, "venv/bin/python extract_gettext.py > conf/strings.js", ) fl.mm(ctx)
def create_virtualenv(): with cd("%(box_domain)s"): run("rm -rf venv") run("python3 -m venv venv") run("venv/bin/pip install -U pip wheel setuptools") run("venv/bin/pip install -r requirements.txt")
def restart(): """Restart the server process""" for line in env["box_restart"]: run(line)
def deploy(*args): """Deploys frontend and backend code to the server if the checking step did not report any problems""" execute('check.deploy') step('\nCompiling static sources...') run_local('yarn run prod') step('\nPushing changes...') run_local('git push origin %(box_branch)s') step('\nDeploying new code on server...') with cd('%(box_domain)s'): run('git fetch') run('git reset --hard origin/%(box_branch)s') run('find . -name "*.pyc" -delete') run('venv/bin/pip install -r requirements.txt') run('venv/bin/python manage.py migrate --noinput') step('\nUploading static files...') rsync_project( local_dir='static/', remote_dir='%(box_domain)s/static/' % env, delete=('clear' in args), ) step('\nCollecting static files...') with cd('%(box_domain)s'): run('venv/bin/python manage.py collectstatic --noinput') step('\nRunning system checks on server...') with cd('%(box_domain)s'): run('venv/bin/python manage.py check --deploy') step('\nRestarting server process...') for line in env['box_restart']: run(line) execute('git.fetch_remote')
def _restart_all(conn): for wb in fl.config.installations: fl.run(conn, f"systemctl --user restart workbench@{wb}", echo=True)
def create_virtualenv(): with cd('%(box_domain)s'): run('rm -rf venv') run('python3 -m venv venv') run('venv/bin/pip install -U pip wheel') run('venv/bin/pip install -r requirements.txt')
def dbshell(): # env.box_remote_db = remote_env('DATABASE_URL') # ssh SERVER -o RequestTTY=yes\ # 'psql $(grep -E "^DATABASE_URL=" %(box_domain)s/.env | cut -f2 -d=)' run("psql %(box_database)s")
def ssl(): run("sudo nine-manage-vhosts certificate create" " --virtual-host=%(box_domain)s") run( "sudo nine-manage-vhosts virtual-host update %(box_domain)s" " --template=%(box_vhost_ssl_template)s" )
def _do_deploy(args=()): with cd("%(box_domain)s"): run('find . -name "*.pyc" -delete') run("venv/bin/pip install -U pip wheel setuptools") run("venv/bin/pip install -r requirements.txt") run("venv/bin/python manage.py migrate --noinput") step("\nUploading static files...") rsync_project( local_dir="static/", remote_dir="%(box_domain)s/static/" % env, delete=("clear" in args), ) step("\nCollecting static files...") with cd("%(box_domain)s"): run("venv/bin/python manage.py collectstatic --noinput") step("\nRunning system checks on server...") with cd("%(box_domain)s"): run("venv/bin/python manage.py check --deploy") step("\nRestarting server process...") execute("server.restart") execute("git.fetch_remote")
def copy_data_from(environment=None): """ Copy the database from one environment to another. Usually from production to stage. Usage: ``fab s server.copy_data_from:production``. :param environment: the source environment """ if env.get('box_hardwired_environment'): abort(red('Cannot continue with a hardwired environment.')) if environment not in env.box_environments: abort(red('Invalid environment %s.' % environment)) source = env.box_environments[environment] target = env.box_environments[env.get('box_environment')] if source == target: abort( red('Source environment %s must not equal target environment %s.' % (environment, env.get('box_environment')))) if source['servers'][0] != target['servers'][0]: abort(red('The environments have to be on the same server, sorry!')) puts('Copying data from {0} to {1}'.format(source['remote'], target['remote'])) if not confirm( 'Completely replace the remote database' ' "%(box_database)s" (if it exists)?', default=False): return for key, value in source.items(): env['source_%s' % key] = value with settings(warn_only=True): run('dropdb %(box_database)s') run('createdb %(box_database)s --encoding=UTF8 --template=template0' ' --owner=%(box_database)s') run('pg_dump -Ox %(source_database)s | psql %(box_database)s') run('psql %(box_database)s -c "REASSIGN OWNED BY admin ' ' TO %(box_database)s"') with cd(env.box_domain): run('cp -aln ~/%(source_domain)s/media/* media/') for line in env['box_restart']: run(line)