def apply_services(name: str, browser_deployment: str = None, reads_deployment: str = None) -> None: if browser_deployment: if not k8s_deployment_exists(f"gnomad-browser-{browser_deployment}"): raise RuntimeError( f"browser deployment {browser_deployment} not found") else: browser_deployment = get_most_recent_k8s_deployment( "component=gnomad-browser")[len("gnomad-browser-"):] if reads_deployment: if not k8s_deployment_exists(f"gnomad-reads-{reads_deployment}"): raise RuntimeError( f"reads deployment {reads_deployment} not found") else: reads_deployment = get_most_recent_k8s_deployment( "component=gnomad-reads")[len("gnomad-reads-"):] manifest = SERVICES_MANIFEST_TEMPLATE.format( name=name, browser_deployment=browser_deployment, reads_deployment=reads_deployment) kubectl(["apply", "-f", "-"], input=manifest)
def apply_deployment(name: str) -> None: deployment_directory = os.path.join(deployments_directory(), name) if not os.path.exists(deployment_directory): raise RuntimeError(f"no configuration for deployment '{name}'") kubectl(["apply", "-k", deployment_directory])
def apply_deployment() -> None: deployment_directory = get_deployment_directory() if not os.path.exists(deployment_directory): raise RuntimeError("no configuration for blog deployment") kubectl(["apply", "-k", deployment_directory])
def apply_ingress(name: str, browser_deployment: str = None, reads_deployment: str = None) -> None: apply_services(name, browser_deployment, reads_deployment) manifest = INGRESS_MANIFEST_TEMPLATE.format(name=name) kubectl(["apply", "-f", "-"], input=manifest)
def create_configmap(): # Store the IP address used for the ingress load balancer in a configmap so that the browser # can use it for determining the real client IP. ingress_ip = gcloud( ["compute", "addresses", "describe", config.ip_address_name, "--global", "--format=value(address)"] ) kubectl(["create", "configmap", "ingress-ip", f"--from-literal=ip={ingress_ip}"])
def delete_deployment(name: str, clean: bool = False) -> None: deployment_directory = os.path.join(deployments_directory(), name) if os.path.exists(deployment_directory): kubectl(["delete", "-k", deployment_directory]) if clean: clean_deployment(name) else: create_deployment(name) delete_deployment(name, clean=True)
def render_template_and_apply( template_path: str, context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> None: if not context: context = {} with open(template_path) as template_file: template = jinja2.Template(template_file.read()) manifest = template.render(**context) kubectl(["apply", "-f", "-"], input=manifest)
def describe_services() -> None: browser_manifest = json.loads( kubectl(["get", "service", "gnomad-browser", "--output=json"])) reads_manifest = json.loads( kubectl(["get", "service", "gnomad-reads", "--output=json"])) browser_deployment = browser_manifest["spec"]["selector"]["deployment"] reads_deployment = reads_manifest["spec"]["selector"]["deployment"] print("active browser deployment:", browser_deployment) print("active reads deployment:", reads_deployment)
def apply_ingress(browser_deployment: str = None, reads_deployment: str = None) -> None: apply_services(browser_deployment, reads_deployment) if input("Apply changes to production ingress (y/n) ").lower() == "y": kubectl([ "apply", "-f", os.path.join(manifests_directory(), "gnomad.managedcertificate.yaml") ]) kubectl([ "apply", "-f", os.path.join(manifests_directory(), "gnomad.ingress.yaml") ])
def get_elasticsearch_password(cluster_name: str) -> None: # ECK creates this secret when the cluster is created. print( kubectl([ "get", "secret", f"{cluster_name}-es-elastic-user", "-o=go-template={{.data.elastic | base64decode}}" ]))
def describe_services(name: str) -> None: try: browser_manifest = json.loads( kubectl([ "get", "service", f"gnomad-browser-demo-{name}", "--output=json" ])) reads_manifest = json.loads( kubectl([ "get", "service", f"gnomad-reads-demo-{name}", "--output=json" ])) browser_deployment = browser_manifest["spec"]["selector"]["deployment"] reads_deployment = reads_manifest["spec"]["selector"]["deployment"] print("active browser deployment:", browser_deployment) print("active reads deployment:", reads_deployment) except Exception: # pylint: disable=broad-except print(f"Could not get services for '{name}' demo environment")
def list_demo_ingresses() -> None: ingresses = kubectl([ "get", "ingresses", "--selector=tier=demo", "--sort-by={.metadata.creationTimestamp}", "--output=jsonpath={range .items[*]}{.metadata.name}{'\\n'}", ]).splitlines() for ingress in ingresses: print(ingress[len("gnomad-ingress-demo-"):])
def create_configmap(): # Store a list of all IP addresses involved in proxying requests. # These are used for determining the real client IP. ingress_ip = gcloud([ "compute", "addresses", "describe", config.ip_address_name, "--global", "--format=value(address)" ]) # Private/internal networks # These ranges match those used for the gnomad-gke subnet. # 127.0.0.1 # 192.168.0.0/20 # 10.4.0.0/14 # 10.0.32.0/20 # # Internal IPs for GCE load balancers # https://cloud.google.com/load-balancing/docs/https#how-connections-work # 35.191.0.0/16 # 130.211.0.0/22 ips = f"127.0.0.1,192.168.0.0/20,10.4.0.0/14,10.0.32.0/20,35.191.0.0/16,130.211.0.0/22,{ingress_ip}" kubectl(["create", "configmap", "proxy-ips", f"--from-literal=ips={ips}"])
def main(argv: typing.List[str]) -> None: parser = argparse.ArgumentParser(prog="deployctl") parser.parse_args(argv) if not config.project: print("project configuration is required", file=sys.stderr) sys.exit(1) print("This will create the following resources:") print(f"- VPC network '{config.network_name}'") print(f"- IP address '{config.ip_address_name}'") print(f"- Service account '{config.gke_service_account_name}'") print(f"- GKE cluster '{config.gke_cluster_name}'") if input("Continue? (y/n) ").lower() == "y": print("Creating network...") create_network() print("Reserving IP address...") create_ip_address() print("Creating service account...") create_cluster_service_account() print("Creating cluster...") create_cluster() print("Creating configmap...") create_configmap() print("Creating node pools...") create_node_pool("redis", ["--num-nodes=1", "--machine-type=n1-highmem-8"]) print("Creating K8S resources...") manifests_directory = os.path.realpath( os.path.join(os.path.dirname(__file__), "../../manifests")) kubectl(["apply", "-k", os.path.join(manifests_directory, "redis")])
def load_datasets(cluster_name: str, dataproc_cluster: str, secret: str, datasets: str): # Matches service name in deploy/manifests/elasticsearch.load-balancer.yaml.jinja2 elasticsearch_load_balancer_ip = kubectl([ "get", "service", f"{cluster_name}-elasticsearch-lb", "--output=jsonpath={.status.loadBalancer.ingress[0].ip}" ]) subprocess.check_call([ sys.argv[0], "data-pipeline", "run", "export_to_elasticsearch", f"--cluster={dataproc_cluster}", "--", f"--host={elasticsearch_load_balancer_ip}", f"--secret={secret}", f"--datasets={datasets}", ])
def get_elasticsearch_cluster(cluster_name: str) -> None: print(kubectl(["get", "elasticsearch", cluster_name]), end="")
def main(argv: typing.List[str]) -> None: parser = argparse.ArgumentParser(prog="deployctl") parser.parse_args(argv) if not config.project: print("project configuration is required", file=sys.stderr) sys.exit(1) print("This will create the following resources:") print(f"- VPC network '{config.network_name}'") print(f"- IP address '{config.ip_address_name}'") print(f"- Router '{config.network_name}-nat-router'") print(f"- NAT config '{config.network_name}-nat'") print(f"- Service account '{config.gke_service_account_name}'") print(f"- GKE cluster '{config.gke_cluster_name}'") print("- Service account 'gnomad-es-snapshots'") print("- Service account 'gnomad-data-pipeline'") if input("Continue? (y/n) ").lower() == "y": print("Creating network...") create_network() print("Reserving IP address...") create_ip_address() print("Creating service account...") create_cluster_service_account() print("Creating cluster...") create_cluster() print("Creating configmap...") create_configmap() print("Creating node pools...") create_node_pool("redis", ["--num-nodes=1", "--machine-type=e2-custom-6-49152"]) create_node_pool("es-data", ["--machine-type=e2-highmem-8"]) print("Creating K8S resources...") manifests_directory = os.path.realpath( os.path.join(os.path.dirname(__file__), "../../manifests")) kubectl(["apply", "-k", os.path.join(manifests_directory, "redis")]) # Install Elastic Cloud on Kubernetes operator # https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-overview.html kubectl([ "apply", "-f", "https://download.elastic.co/downloads/eck/1.2.1/all-in-one.yaml" ]) # Configure firewall rule for ECK admission webhook # https://github.com/elastic/cloud-on-k8s/issues/1437 # https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules gke_firewall_rule_target_tags = gcloud([ "compute", "firewall-rules", "list", f"--filter=name~^gke-{config.gke_cluster_name}", "--format=value(targetTags.list())", ]).splitlines()[0] gcloud([ "compute", "firewall-rules", "create", f"{config.network_name}-es-webhook", "--action=ALLOW", "--direction=INGRESS", f"--network={config.network_name}", "--rules=tcp:9443", "--source-ranges=172.16.0.0/28", # Matches GKE cluster master IP range f"--target-tags={gke_firewall_rule_target_tags}", ]) # Create a service account for Elasticsearch snapshots # https://www.elastic.co/guide/en/cloud-on-k8s/1.2/k8s-snapshots.html#k8s-secure-settings try: # Do not alter the service account if it already exists. # Deleting and recreating a service account with the same name can lead to unexpected behavior # https://cloud.google.com/iam/docs/understanding-service-accounts#deleting_and_recreating_service_accounts gcloud( [ "iam", "service-accounts", "describe", f"gnomad-es-snapshots@{config.project}.iam.gserviceaccount.com", ], stderr=subprocess.DEVNULL, ) print("Snapshot account already exists") except subprocess.CalledProcessError: gcloud([ "iam", "service-accounts", "create", "gnomad-es-snapshots", "--display-name=gnomAD Elasticsearch snapshots", ]) finally: # Grant the snapshot service account object admin access to the snapshot bucket. # https://cloud.google.com/storage/docs/access-control/using-iam-permissions#bucket-add subprocess.check_call( [ "gsutil", "iam", "ch", f"serviceAccount:gnomad-es-snapshots@{config.project}.iam.gserviceaccount.com:roles/storage.admin", "gs://gnomad-browser-elasticsearch-snapshots", # TODO: The bucket to use for snapshots should be configurable ], stdout=subprocess.DEVNULL, ) # Download key for snapshots service account. # https://cloud.google.com/iam/docs/creating-managing-service-account-keys keys_directory = os.path.realpath( os.path.join(os.path.dirname(__file__), "../../keys")) if not os.path.exists(keys_directory): os.mkdir(keys_directory) with open(os.path.join(keys_directory, ".gitignore"), "w") as gitignore_file: gitignore_file.write("*") if not os.path.exists( os.path.join(keys_directory, "gcs.client.default.credentials_file")): gcloud([ "iam", "service-accounts", "keys", "create", os.path.join(keys_directory, "gcs.client.default.credentials_file"), f"--iam-account=gnomad-es-snapshots@{config.project}.iam.gserviceaccount.com", ]) # Create K8S secret with snapshots service account key. kubectl( [ "create", "secret", "generic", "es-snapshots-gcs-credentials", "--from-file=gcs.client.default.credentials_file", ], cwd=keys_directory, ) # Create a service account for data pipeline. try: # Do not alter the service account if it already exists. # Deleting and recreating a service account with the same name can lead to unexpected behavior # https://cloud.google.com/iam/docs/understanding-service-accounts#deleting_and_recreating_service_accounts gcloud( [ "iam", "service-accounts", "describe", f"gnomad-data-pipeline@{config.project}.iam.gserviceaccount.com", ], stderr=subprocess.DEVNULL, ) print("Data pipeline service account already exists") except subprocess.CalledProcessError: gcloud([ "iam", "service-accounts", "create", "gnomad-data-pipeline", "--display-name=gnomAD data pipeline" ]) # Grant the data pipeline service account the Dataproc worker role. subprocess.check_call( [ "gcloud", "projects", "add-iam-policy-binding", config.project, f"--member=serviceAccount:gnomad-data-pipeline@{config.project}.iam.gserviceaccount.com", "--role=roles/dataproc.worker", ], stdout=subprocess.DEVNULL, ) # serviceusage.services.use is necessary to access requester pays buckets subprocess.check_call( [ "gcloud", "projects", "add-iam-policy-binding", config.project, f"--member=serviceAccount:gnomad-data-pipeline@{config.project}.iam.gserviceaccount.com", "--role=roles/roles/serviceusage.serviceUsageConsumer", ], stdout=subprocess.DEVNULL, ) finally: # Grant the data pipeline service account object admin access to the data pipeline bucket. # https://cloud.google.com/storage/docs/access-control/using-iam-permissions#bucket-add subprocess.check_call( [ "gsutil", "iam", "ch", f"serviceAccount:gnomad-data-pipeline@{config.project}.iam.gserviceaccount.com:roles/storage.admin", # TODO: This should use the same configuration as data pipeline output. "gs://gnomad-browser-data-pipeline", ], stdout=subprocess.DEVNULL, )
def delete_ingress_and_services(name: str) -> None: kubectl(["delete", f"ingress/gnomad-ingress-demo-{name}"]) kubectl(["delete", f"service/gnomad-reads-demo-{name}"]) kubectl(["delete", f"service/gnomad-browser-demo-{name}"])