def redis_secret(ctx, host='10.0.0.3', port=6379, db=None, namespace=TEST_NAMESPACE): """Create a secret containing redis credentials""" # limit usage of reserved dbs if namespace in REDIS_RESERVED_DB.keys(): db = REDIS_RESERVED_DB[namespace] print(f"using reserved Redis DB '{db}' for namespace '{namespace}'") if db in REDIS_RESERVED_DB.values() and REDIS_RESERVED_DB[namespace] != db: raise ValueError( f"namespace {namespace} cannot use reserved DB id {db}") # come up with a consistent hash from the namespace, exclude reserved values if db is None: min_available = max(REDIS_RESERVED_DB.values()) + 1 namespace_hash = int(md5(namespace.encode('utf-8')).hexdigest(), 16) db = (namespace_hash % 12) + min_available print( f"Namespace '{namespace}' got assigned Redis DB '{db}' -- this might not be unique!" ) secret = load_manifest('redis_secret', { 'host': host, 'port': str(port), 'db': str(db), }) with K8SNamespace(namespace) as ns: ns.apply(secret)
def run_migrations(ctx, namespace=TEST_NAMESPACE): """Migrate the DB that the cluster is connected to""" with K8SNamespace(namespace) as ns: random_pod_name = ns.random_pod_name() print(f"Using pod {random_pod_name} to run migrations...") run(f'{ns.kubecmd} exec {random_pod_name} inv run.upgrade')
def pods(ctx, namespace=TEST_NAMESPACE): """Display existing pods for the service""" print('Phase\tImage\tName') print('------------------------------') with K8SNamespace(namespace) as ns: for pod in ns.get_pods(): print(f'{pod["phase"]}\t{pod["image"]}\t{pod["name"]}')
def shell(ctx, namespace=TEST_NAMESPACE): """Get a shell on a random pod""" with K8SNamespace(namespace) as ns: random_pod_name = ns.random_pod_name() print(f"Picked pod {random_pod_name} for shell access...") run(f'{ns.kubecmd} exec -i -t {random_pod_name} -- /bin/bash', pty=True)
def bounce(ctx, namespace=TEST_NAMESPACE): """Bounce the deployment by deleting all pods and allowing them to be recreated""" with K8SNamespace(namespace) as ns: pod_names = [ p['name'] for p in ns.get_pods() if p['phase'] == 'Running' ] for pod_name in pod_names: run(f'{ns.kubecmd} delete pod {pod_name}')
def sendgrid_secret(ctx, secret_key, namespace=TEST_NAMESPACE): """The secret to sending email""" secret = load_manifest('sendgrid_secret', { 'secret_key': secret_key, }) with K8SNamespace(namespace) as ns: ns.apply(secret)
def session_secret(ctx, secret_key=None, namespace=TEST_NAMESPACE): """Creates a new session secret; this will invalidate existing sessions""" if not secret_key: secret_key = random_string(24) secret = load_manifest('session_secret', { 'secret_key': secret_key, }) with K8SNamespace(namespace) as ns: ns.apply(secret)
def google_app_secret(ctx, keyfile, namespace=TEST_NAMESPACE): """upload a keyfile as a google secret created by: gcloud iam service-accounts keys create /tmp/key.json --iam-account=pyspaceship@spaceshipearthprod.iam.gserviceaccount.com """ secret = load_manifest('google-app-creds', { 'keyfile': open(keyfile).read().strip(), }) with K8SNamespace(namespace) as ns: ns.apply(secret)
def mysql_secret(ctx, password, host='10.82.64.3', port=3306, username='******', db='spaceship', namespace=TEST_NAMESPACE): """Create a secret containing mysql credentials""" secret = load_manifest( 'mysql_secret', { 'host': host, 'port': str(port), 'username': username, 'password': password, 'db': db, }) with K8SNamespace(namespace) as ns: ns.apply(secret)
def release(ctx, version=None, namespace=TEST_NAMESPACE): """Releases a new version of the site""" # default to a hash of the repo state if not version: version = get_hash() print(f"pushing image using version {version}") # sanity check on the specified version if not (version.startswith('v') and len(version.split('.')) == 3): print( f"Specified version {version} doesn't look production-y (like, 'v1.2.3'), so skipping git tagging" ) do_tag = False else: do_tag = True # pull to get the latest list of existing tags run('git fetch --tags') # check for tag conflicts existing_git_tags = run('git tag --list', hide=True).stdout.split('\n') if version in existing_git_tags: raise Exit( "There is already a commit tagged with version %s -- use a later version!" % version) # generate and push correctly-tagged build docker_tag = generate_tag(version) with in_repo_root(): do_build(docker_tag) do_push(docker_tag) # mark the git repo as corresponding to that tag if do_tag: run('git tag -a %s -m "Releasing image %s"' % (version, docker_tag)) run('git push --tags') # do the deploy with K8SNamespace(namespace) as ns: do_deploy(docker_tag, ns)
def do_deploy(tag, ns: K8SNamespace): """actually perform a deploy of the manifests""" replicas = 1 if ns.is_prod: replicas = 3 context = { 'namespace': ns.namespace, 'image': tag, 'replicas': replicas, } context['container_environment'] = load_manifest('container_environment', context) web = load_manifest('web_deployment', context) ns.apply(web) worker = load_manifest('worker_deployment', context) ns.apply(worker) service = load_manifest('service', context) ns.apply(service)
def namespace(ctx, namespace): """initializes a namespace, like a site version""" # create a db with a user for the namespace user = f"spaceship-{namespace}" try: run(f"gcloud sql databases create {user} --instance=spaceshipdb", hide=True) print(f"created db {user}") except UnexpectedExit as e: if 'database exists' in e.result.stderr: print(f'db {user} already existed') else: raise # create the namespace with K8SNamespace.prod() as prod: nsmanifest = load_manifest('namespace', {'namespace': namespace}) prod.apply(nsmanifest) # create the db user and session secrets if they don't exist with K8SNamespace(namespace) as ns: existing = ns.get_secret('pyspaceship-mysql') if not existing: password = random_string(12) run(f"gcloud sql users create {user} --host='%' --instance=spaceshipdb --password={password}", hide=True) print(f'created db user {user}') mysql_secret(ctx, username=user, password=password, namespace=namespace, db=user) else: print('mysql user already initialized') existing = ns.get_secret('pyspaceship-session') if not existing: session_secret(ctx, namespace=namespace) else: print('session already initialized') existing = ns.get_secret('pyspaceship-redis') if not existing: redis_secret(ctx, namespace=namespace) else: print('redis already initialized') # copy the google and sendgrid secrets from prod for secret_name in [ 'pyspaceship-google-oauth', 'pyspaceship-sendgrid', 'google-app-creds' ]: with K8SNamespace.prod() as prod: data = prod.get_secret(secret_name) secret = load_manifest('generic_secret', { 'name': secret_name, 'data': data }) with K8SNamespace(namespace) as ns: ns.apply(secret) # create an ssl cert and ingress for the namespace with K8SNamespace(namespace) as ns: ssl_cert = load_manifest('cert', {'namespace': namespace}) ns.apply(ssl_cert) ingress = load_manifest('ingress', {'namespace': namespace}) ns.apply(ingress) info = json.loads( run(f'{ns.kubecmd} get ingress pyspaceship-ingress -o=json', hide=True).stdout) print("Load balancer IPs:") try: ingress = info['status']['loadBalancer']['ingress'] except KeyError: print("None assigned (yet?) -- re-run later to get IP address") else: ips = set() for i in ingress: ip = i['ip'] ips.add(ip) print(f'- {ip}') dns_name = f'{namespace}.spaceshipearth.org.' existing = set() existing_data = json.loads( run(f'gcloud dns record-sets list -z spaceshipearth-org --name "{dns_name}" --format json', hide=True).stdout) for record in existing_data: for addr in record['rrdatas']: existing.add(addr) to_add = ips - existing if len(to_add) == 0: print('all ip addresses already in DNS') for addr in to_add: print(f'adding {addr} as an A record for {dns_name}') def dns_trans(cmd): base_cmd = 'gcloud dns record-sets transaction' suffix = '--zone spaceshipearth-org' return run(f"{base_cmd} {cmd} {suffix}", hide=True) dns_trans('start') dns_trans( f'add --name="{dns_name}" --type=A --ttl 300 "{addr}"') dns_trans('execute')
def show_secret(ctx, secret_name, namespace=TEST_NAMESPACE): """Displays the contents of a kubernetes secret""" with K8SNamespace(namespace) as ns: data = ns.get_secret(secret_name) print(json.dumps(data, indent=4))
def deploy(ctx, version, namespace=TEST_NAMESPACE, dry_run=False): """Generates and applies k8s configuration (second part of `release`)""" tag = generate_tag(version) with K8SNamespace(namespace, dry_run) as ns: do_deploy(tag, ns)