def redis_secret(ctx,
host='10.0.0.3',
port=6379,
db=None,
namespace=TEST_NAMESPACE):
"""Create a secret containing redis credentials"""
# limit usage of reserved dbs
if namespace in REDIS_RESERVED_DB.keys():
db = REDIS_RESERVED_DB[namespace]
print(f"using reserved Redis DB '{db}' for namespace '{namespace}'")
if db in REDIS_RESERVED_DB.values() and REDIS_RESERVED_DB[namespace] != db:
raise ValueError(
f"namespace {namespace} cannot use reserved DB id {db}")
# come up with a consistent hash from the namespace, exclude reserved values
if db is None:
min_available = max(REDIS_RESERVED_DB.values()) + 1
namespace_hash = int(md5(namespace.encode('utf-8')).hexdigest(), 16)
db = (namespace_hash % 12) + min_available
print(
f"Namespace '{namespace}' got assigned Redis DB '{db}' -- this might not be unique!"
)
secret = load_manifest('redis_secret', {
'host': host,
'port': str(port),
'db': str(db),
})
with K8SNamespace(namespace) as ns:
ns.apply(secret)
def run_migrations(ctx, namespace=TEST_NAMESPACE):
"""Migrate the DB that the cluster is connected to"""
with K8SNamespace(namespace) as ns:
random_pod_name = ns.random_pod_name()
print(f"Using pod {random_pod_name} to run migrations...")
run(f'{ns.kubecmd} exec {random_pod_name} inv run.upgrade')
def pods(ctx, namespace=TEST_NAMESPACE):
"""Display existing pods for the service"""
print('Phase\tImage\tName')
print('------------------------------')
with K8SNamespace(namespace) as ns:
for pod in ns.get_pods():
print(f'{pod["phase"]}\t{pod["image"]}\t{pod["name"]}')
def shell(ctx, namespace=TEST_NAMESPACE):
"""Get a shell on a random pod"""
with K8SNamespace(namespace) as ns:
random_pod_name = ns.random_pod_name()
print(f"Picked pod {random_pod_name} for shell access...")
run(f'{ns.kubecmd} exec -i -t {random_pod_name} -- /bin/bash',
pty=True)
def bounce(ctx, namespace=TEST_NAMESPACE):
"""Bounce the deployment by deleting all pods and allowing them to be recreated"""
with K8SNamespace(namespace) as ns:
pod_names = [
p['name'] for p in ns.get_pods() if p['phase'] == 'Running'
]
for pod_name in pod_names:
run(f'{ns.kubecmd} delete pod {pod_name}')
def sendgrid_secret(ctx, secret_key, namespace=TEST_NAMESPACE):
"""The secret to sending email"""
secret = load_manifest('sendgrid_secret', {
'secret_key': secret_key,
})
with K8SNamespace(namespace) as ns:
ns.apply(secret)
def session_secret(ctx, secret_key=None, namespace=TEST_NAMESPACE):
"""Creates a new session secret; this will invalidate existing sessions"""
if not secret_key:
secret_key = random_string(24)
secret = load_manifest('session_secret', {
'secret_key': secret_key,
})
with K8SNamespace(namespace) as ns:
ns.apply(secret)
def google_app_secret(ctx, keyfile, namespace=TEST_NAMESPACE):
"""upload a keyfile as a google secret
created by:
gcloud iam service-accounts keys create /tmp/key.json --iam-account=pyspaceship@spaceshipearthprod.iam.gserviceaccount.com
"""
secret = load_manifest('google-app-creds', {
'keyfile': open(keyfile).read().strip(),
})
with K8SNamespace(namespace) as ns:
ns.apply(secret)
def mysql_secret(ctx,
password,
host='10.82.64.3',
port=3306,
username='******',
db='spaceship',
namespace=TEST_NAMESPACE):
"""Create a secret containing mysql credentials"""
secret = load_manifest(
'mysql_secret', {
'host': host,
'port': str(port),
'username': username,
'password': password,
'db': db,
})
with K8SNamespace(namespace) as ns:
ns.apply(secret)
def release(ctx, version=None, namespace=TEST_NAMESPACE):
"""Releases a new version of the site"""
# default to a hash of the repo state
if not version:
version = get_hash()
print(f"pushing image using version {version}")
# sanity check on the specified version
if not (version.startswith('v') and len(version.split('.')) == 3):
print(
f"Specified version {version} doesn't look production-y (like, 'v1.2.3'), so skipping git tagging"
)
do_tag = False
else:
do_tag = True
# pull to get the latest list of existing tags
run('git fetch --tags')
# check for tag conflicts
existing_git_tags = run('git tag --list', hide=True).stdout.split('\n')
if version in existing_git_tags:
raise Exit(
"There is already a commit tagged with version %s -- use a later version!"
% version)
# generate and push correctly-tagged build
docker_tag = generate_tag(version)
with in_repo_root():
do_build(docker_tag)
do_push(docker_tag)
# mark the git repo as corresponding to that tag
if do_tag:
run('git tag -a %s -m "Releasing image %s"' % (version, docker_tag))
run('git push --tags')
# do the deploy
with K8SNamespace(namespace) as ns:
do_deploy(docker_tag, ns)
def do_deploy(tag, ns: K8SNamespace):
"""actually perform a deploy of the manifests"""
replicas = 1
if ns.is_prod:
replicas = 3
context = {
'namespace': ns.namespace,
'image': tag,
'replicas': replicas,
}
context['container_environment'] = load_manifest('container_environment',
context)
web = load_manifest('web_deployment', context)
ns.apply(web)
worker = load_manifest('worker_deployment', context)
ns.apply(worker)
service = load_manifest('service', context)
ns.apply(service)
def namespace(ctx, namespace):
"""initializes a namespace, like a site version"""
# create a db with a user for the namespace
user = f"spaceship-{namespace}"
try:
run(f"gcloud sql databases create {user} --instance=spaceshipdb",
hide=True)
print(f"created db {user}")
except UnexpectedExit as e:
if 'database exists' in e.result.stderr:
print(f'db {user} already existed')
else:
raise
# create the namespace
with K8SNamespace.prod() as prod:
nsmanifest = load_manifest('namespace', {'namespace': namespace})
prod.apply(nsmanifest)
# create the db user and session secrets if they don't exist
with K8SNamespace(namespace) as ns:
existing = ns.get_secret('pyspaceship-mysql')
if not existing:
password = random_string(12)
run(f"gcloud sql users create {user} --host='%' --instance=spaceshipdb --password={password}",
hide=True)
print(f'created db user {user}')
mysql_secret(ctx,
username=user,
password=password,
namespace=namespace,
db=user)
else:
print('mysql user already initialized')
existing = ns.get_secret('pyspaceship-session')
if not existing:
session_secret(ctx, namespace=namespace)
else:
print('session already initialized')
existing = ns.get_secret('pyspaceship-redis')
if not existing:
redis_secret(ctx, namespace=namespace)
else:
print('redis already initialized')
# copy the google and sendgrid secrets from prod
for secret_name in [
'pyspaceship-google-oauth', 'pyspaceship-sendgrid',
'google-app-creds'
]:
with K8SNamespace.prod() as prod:
data = prod.get_secret(secret_name)
secret = load_manifest('generic_secret', {
'name': secret_name,
'data': data
})
with K8SNamespace(namespace) as ns:
ns.apply(secret)
# create an ssl cert and ingress for the namespace
with K8SNamespace(namespace) as ns:
ssl_cert = load_manifest('cert', {'namespace': namespace})
ns.apply(ssl_cert)
ingress = load_manifest('ingress', {'namespace': namespace})
ns.apply(ingress)
info = json.loads(
run(f'{ns.kubecmd} get ingress pyspaceship-ingress -o=json',
hide=True).stdout)
print("Load balancer IPs:")
try:
ingress = info['status']['loadBalancer']['ingress']
except KeyError:
print("None assigned (yet?) -- re-run later to get IP address")
else:
ips = set()
for i in ingress:
ip = i['ip']
ips.add(ip)
print(f'- {ip}')
dns_name = f'{namespace}.spaceshipearth.org.'
existing = set()
existing_data = json.loads(
run(f'gcloud dns record-sets list -z spaceshipearth-org --name "{dns_name}" --format json',
hide=True).stdout)
for record in existing_data:
for addr in record['rrdatas']:
existing.add(addr)
to_add = ips - existing
if len(to_add) == 0:
print('all ip addresses already in DNS')
for addr in to_add:
print(f'adding {addr} as an A record for {dns_name}')
def dns_trans(cmd):
base_cmd = 'gcloud dns record-sets transaction'
suffix = '--zone spaceshipearth-org'
return run(f"{base_cmd} {cmd} {suffix}", hide=True)
dns_trans('start')
dns_trans(
f'add --name="{dns_name}" --type=A --ttl 300 "{addr}"')
dns_trans('execute')
def show_secret(ctx, secret_name, namespace=TEST_NAMESPACE):
"""Displays the contents of a kubernetes secret"""
with K8SNamespace(namespace) as ns:
data = ns.get_secret(secret_name)
print(json.dumps(data, indent=4))
def deploy(ctx, version, namespace=TEST_NAMESPACE, dry_run=False):
"""Generates and applies k8s configuration (second part of `release`)"""
tag = generate_tag(version)
with K8SNamespace(namespace, dry_run) as ns:
do_deploy(tag, ns)