def apply_services(name: str,
browser_deployment: str = None,
reads_deployment: str = None) -> None:
if browser_deployment:
if not k8s_deployment_exists(f"gnomad-browser-{browser_deployment}"):
raise RuntimeError(
f"browser deployment {browser_deployment} not found")
else:
browser_deployment = get_most_recent_k8s_deployment(
"component=gnomad-browser")[len("gnomad-browser-"):]
if reads_deployment:
if not k8s_deployment_exists(f"gnomad-reads-{reads_deployment}"):
raise RuntimeError(
f"reads deployment {reads_deployment} not found")
else:
reads_deployment = get_most_recent_k8s_deployment(
"component=gnomad-reads")[len("gnomad-reads-"):]
manifest = SERVICES_MANIFEST_TEMPLATE.format(
name=name,
browser_deployment=browser_deployment,
reads_deployment=reads_deployment)
kubectl(["apply", "-f", "-"], input=manifest)
def apply_deployment(name: str) -> None:
deployment_directory = os.path.join(deployments_directory(), name)
if not os.path.exists(deployment_directory):
raise RuntimeError(f"no configuration for deployment '{name}'")
kubectl(["apply", "-k", deployment_directory])
def apply_deployment() -> None:
deployment_directory = get_deployment_directory()
if not os.path.exists(deployment_directory):
raise RuntimeError("no configuration for blog deployment")
kubectl(["apply", "-k", deployment_directory])
def apply_ingress(name: str,
browser_deployment: str = None,
reads_deployment: str = None) -> None:
apply_services(name, browser_deployment, reads_deployment)
manifest = INGRESS_MANIFEST_TEMPLATE.format(name=name)
kubectl(["apply", "-f", "-"], input=manifest)
def create_configmap():
# Store the IP address used for the ingress load balancer in a configmap so that the browser
# can use it for determining the real client IP.
ingress_ip = gcloud(
["compute", "addresses", "describe", config.ip_address_name, "--global", "--format=value(address)"]
)
kubectl(["create", "configmap", "ingress-ip", f"--from-literal=ip={ingress_ip}"])
def delete_deployment(name: str, clean: bool = False) -> None:
deployment_directory = os.path.join(deployments_directory(), name)
if os.path.exists(deployment_directory):
kubectl(["delete", "-k", deployment_directory])
if clean:
clean_deployment(name)
else:
create_deployment(name)
delete_deployment(name, clean=True)
def render_template_and_apply(
template_path: str,
context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> None:
if not context:
context = {}
with open(template_path) as template_file:
template = jinja2.Template(template_file.read())
manifest = template.render(**context)
kubectl(["apply", "-f", "-"], input=manifest)
def describe_services() -> None:
browser_manifest = json.loads(
kubectl(["get", "service", "gnomad-browser", "--output=json"]))
reads_manifest = json.loads(
kubectl(["get", "service", "gnomad-reads", "--output=json"]))
browser_deployment = browser_manifest["spec"]["selector"]["deployment"]
reads_deployment = reads_manifest["spec"]["selector"]["deployment"]
print("active browser deployment:", browser_deployment)
print("active reads deployment:", reads_deployment)
def apply_ingress(browser_deployment: str = None,
reads_deployment: str = None) -> None:
apply_services(browser_deployment, reads_deployment)
if input("Apply changes to production ingress (y/n) ").lower() == "y":
kubectl([
"apply", "-f",
os.path.join(manifests_directory(),
"gnomad.managedcertificate.yaml")
])
kubectl([
"apply", "-f",
os.path.join(manifests_directory(), "gnomad.ingress.yaml")
])
def get_elasticsearch_password(cluster_name: str) -> None:
# ECK creates this secret when the cluster is created.
print(
kubectl([
"get", "secret", f"{cluster_name}-es-elastic-user",
"-o=go-template={{.data.elastic | base64decode}}"
]))
def describe_services(name: str) -> None:
try:
browser_manifest = json.loads(
kubectl([
"get", "service", f"gnomad-browser-demo-{name}",
"--output=json"
]))
reads_manifest = json.loads(
kubectl([
"get", "service", f"gnomad-reads-demo-{name}", "--output=json"
]))
browser_deployment = browser_manifest["spec"]["selector"]["deployment"]
reads_deployment = reads_manifest["spec"]["selector"]["deployment"]
print("active browser deployment:", browser_deployment)
print("active reads deployment:", reads_deployment)
except Exception: # pylint: disable=broad-except
print(f"Could not get services for '{name}' demo environment")
def list_demo_ingresses() -> None:
ingresses = kubectl([
"get",
"ingresses",
"--selector=tier=demo",
"--sort-by={.metadata.creationTimestamp}",
"--output=jsonpath={range .items[*]}{.metadata.name}{'\\n'}",
]).splitlines()
for ingress in ingresses:
print(ingress[len("gnomad-ingress-demo-"):])
def create_configmap():
# Store a list of all IP addresses involved in proxying requests.
# These are used for determining the real client IP.
ingress_ip = gcloud([
"compute", "addresses", "describe", config.ip_address_name, "--global",
"--format=value(address)"
])
# Private/internal networks
# These ranges match those used for the gnomad-gke subnet.
# 127.0.0.1
# 192.168.0.0/20
# 10.4.0.0/14
# 10.0.32.0/20
#
# Internal IPs for GCE load balancers
# https://cloud.google.com/load-balancing/docs/https#how-connections-work
# 35.191.0.0/16
# 130.211.0.0/22
ips = f"127.0.0.1,192.168.0.0/20,10.4.0.0/14,10.0.32.0/20,35.191.0.0/16,130.211.0.0/22,{ingress_ip}"
kubectl(["create", "configmap", "proxy-ips", f"--from-literal=ips={ips}"])
def main(argv: typing.List[str]) -> None:
parser = argparse.ArgumentParser(prog="deployctl")
parser.parse_args(argv)
if not config.project:
print("project configuration is required", file=sys.stderr)
sys.exit(1)
print("This will create the following resources:")
print(f"- VPC network '{config.network_name}'")
print(f"- IP address '{config.ip_address_name}'")
print(f"- Service account '{config.gke_service_account_name}'")
print(f"- GKE cluster '{config.gke_cluster_name}'")
if input("Continue? (y/n) ").lower() == "y":
print("Creating network...")
create_network()
print("Reserving IP address...")
create_ip_address()
print("Creating service account...")
create_cluster_service_account()
print("Creating cluster...")
create_cluster()
print("Creating configmap...")
create_configmap()
print("Creating node pools...")
create_node_pool("redis",
["--num-nodes=1", "--machine-type=n1-highmem-8"])
print("Creating K8S resources...")
manifests_directory = os.path.realpath(
os.path.join(os.path.dirname(__file__), "../../manifests"))
kubectl(["apply", "-k", os.path.join(manifests_directory, "redis")])
def load_datasets(cluster_name: str, dataproc_cluster: str, secret: str,
datasets: str):
# Matches service name in deploy/manifests/elasticsearch.load-balancer.yaml.jinja2
elasticsearch_load_balancer_ip = kubectl([
"get", "service", f"{cluster_name}-elasticsearch-lb",
"--output=jsonpath={.status.loadBalancer.ingress[0].ip}"
])
subprocess.check_call([
sys.argv[0],
"data-pipeline",
"run",
"export_to_elasticsearch",
f"--cluster={dataproc_cluster}",
"--",
f"--host={elasticsearch_load_balancer_ip}",
f"--secret={secret}",
f"--datasets={datasets}",
])
def get_elasticsearch_cluster(cluster_name: str) -> None:
print(kubectl(["get", "elasticsearch", cluster_name]), end="")
def main(argv: typing.List[str]) -> None:
parser = argparse.ArgumentParser(prog="deployctl")
parser.parse_args(argv)
if not config.project:
print("project configuration is required", file=sys.stderr)
sys.exit(1)
print("This will create the following resources:")
print(f"- VPC network '{config.network_name}'")
print(f"- IP address '{config.ip_address_name}'")
print(f"- Router '{config.network_name}-nat-router'")
print(f"- NAT config '{config.network_name}-nat'")
print(f"- Service account '{config.gke_service_account_name}'")
print(f"- GKE cluster '{config.gke_cluster_name}'")
print("- Service account 'gnomad-es-snapshots'")
print("- Service account 'gnomad-data-pipeline'")
if input("Continue? (y/n) ").lower() == "y":
print("Creating network...")
create_network()
print("Reserving IP address...")
create_ip_address()
print("Creating service account...")
create_cluster_service_account()
print("Creating cluster...")
create_cluster()
print("Creating configmap...")
create_configmap()
print("Creating node pools...")
create_node_pool("redis",
["--num-nodes=1", "--machine-type=e2-custom-6-49152"])
create_node_pool("es-data", ["--machine-type=e2-highmem-8"])
print("Creating K8S resources...")
manifests_directory = os.path.realpath(
os.path.join(os.path.dirname(__file__), "../../manifests"))
kubectl(["apply", "-k", os.path.join(manifests_directory, "redis")])
# Install Elastic Cloud on Kubernetes operator
# https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-overview.html
kubectl([
"apply", "-f",
"https://download.elastic.co/downloads/eck/1.2.1/all-in-one.yaml"
])
# Configure firewall rule for ECK admission webhook
# https://github.com/elastic/cloud-on-k8s/issues/1437
# https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules
gke_firewall_rule_target_tags = gcloud([
"compute",
"firewall-rules",
"list",
f"--filter=name~^gke-{config.gke_cluster_name}",
"--format=value(targetTags.list())",
]).splitlines()[0]
gcloud([
"compute",
"firewall-rules",
"create",
f"{config.network_name}-es-webhook",
"--action=ALLOW",
"--direction=INGRESS",
f"--network={config.network_name}",
"--rules=tcp:9443",
"--source-ranges=172.16.0.0/28", # Matches GKE cluster master IP range
f"--target-tags={gke_firewall_rule_target_tags}",
])
# Create a service account for Elasticsearch snapshots
# https://www.elastic.co/guide/en/cloud-on-k8s/1.2/k8s-snapshots.html#k8s-secure-settings
try:
# Do not alter the service account if it already exists.
# Deleting and recreating a service account with the same name can lead to unexpected behavior
# https://cloud.google.com/iam/docs/understanding-service-accounts#deleting_and_recreating_service_accounts
gcloud(
[
"iam",
"service-accounts",
"describe",
f"gnomad-es-snapshots@{config.project}.iam.gserviceaccount.com",
],
stderr=subprocess.DEVNULL,
)
print("Snapshot account already exists")
except subprocess.CalledProcessError:
gcloud([
"iam",
"service-accounts",
"create",
"gnomad-es-snapshots",
"--display-name=gnomAD Elasticsearch snapshots",
])
finally:
# Grant the snapshot service account object admin access to the snapshot bucket.
# https://cloud.google.com/storage/docs/access-control/using-iam-permissions#bucket-add
subprocess.check_call(
[
"gsutil",
"iam",
"ch",
f"serviceAccount:gnomad-es-snapshots@{config.project}.iam.gserviceaccount.com:roles/storage.admin",
"gs://gnomad-browser-elasticsearch-snapshots", # TODO: The bucket to use for snapshots should be configurable
],
stdout=subprocess.DEVNULL,
)
# Download key for snapshots service account.
# https://cloud.google.com/iam/docs/creating-managing-service-account-keys
keys_directory = os.path.realpath(
os.path.join(os.path.dirname(__file__), "../../keys"))
if not os.path.exists(keys_directory):
os.mkdir(keys_directory)
with open(os.path.join(keys_directory, ".gitignore"),
"w") as gitignore_file:
gitignore_file.write("*")
if not os.path.exists(
os.path.join(keys_directory,
"gcs.client.default.credentials_file")):
gcloud([
"iam",
"service-accounts",
"keys",
"create",
os.path.join(keys_directory,
"gcs.client.default.credentials_file"),
f"--iam-account=gnomad-es-snapshots@{config.project}.iam.gserviceaccount.com",
])
# Create K8S secret with snapshots service account key.
kubectl(
[
"create",
"secret",
"generic",
"es-snapshots-gcs-credentials",
"--from-file=gcs.client.default.credentials_file",
],
cwd=keys_directory,
)
# Create a service account for data pipeline.
try:
# Do not alter the service account if it already exists.
# Deleting and recreating a service account with the same name can lead to unexpected behavior
# https://cloud.google.com/iam/docs/understanding-service-accounts#deleting_and_recreating_service_accounts
gcloud(
[
"iam",
"service-accounts",
"describe",
f"gnomad-data-pipeline@{config.project}.iam.gserviceaccount.com",
],
stderr=subprocess.DEVNULL,
)
print("Data pipeline service account already exists")
except subprocess.CalledProcessError:
gcloud([
"iam", "service-accounts", "create", "gnomad-data-pipeline",
"--display-name=gnomAD data pipeline"
])
# Grant the data pipeline service account the Dataproc worker role.
subprocess.check_call(
[
"gcloud",
"projects",
"add-iam-policy-binding",
config.project,
f"--member=serviceAccount:gnomad-data-pipeline@{config.project}.iam.gserviceaccount.com",
"--role=roles/dataproc.worker",
],
stdout=subprocess.DEVNULL,
)
# serviceusage.services.use is necessary to access requester pays buckets
subprocess.check_call(
[
"gcloud",
"projects",
"add-iam-policy-binding",
config.project,
f"--member=serviceAccount:gnomad-data-pipeline@{config.project}.iam.gserviceaccount.com",
"--role=roles/roles/serviceusage.serviceUsageConsumer",
],
stdout=subprocess.DEVNULL,
)
finally:
# Grant the data pipeline service account object admin access to the data pipeline bucket.
# https://cloud.google.com/storage/docs/access-control/using-iam-permissions#bucket-add
subprocess.check_call(
[
"gsutil",
"iam",
"ch",
f"serviceAccount:gnomad-data-pipeline@{config.project}.iam.gserviceaccount.com:roles/storage.admin",
# TODO: This should use the same configuration as data pipeline output.
"gs://gnomad-browser-data-pipeline",
],
stdout=subprocess.DEVNULL,
)
def delete_ingress_and_services(name: str) -> None:
kubectl(["delete", f"ingress/gnomad-ingress-demo-{name}"])
kubectl(["delete", f"service/gnomad-reads-demo-{name}"])
kubectl(["delete", f"service/gnomad-browser-demo-{name}"])
