def run_in_pod(pod_name, command, deployment_target=None, errors_to_ignore=None, print_command=True, verbose=False, is_interactive=False): """Execute an arbitrary linux command inside the given pod. Assumes there's only 1 instance with the given pod_name. Args: pod_name (str): either the pod's "name" label (eg. 'phenotips' or 'nginx'), or the full pod name (eg. "phenotips-cdd4d7dc9-vgmjx") command (str): linux command to execute inside the pod deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "minikube", "gcloud-dev", etc. errors_to_ignore (list): if the command's return code isn't in ok_return_codes, but its output contains one of the strings in this list, the bad return code will be ignored, and this function will return None. Otherwise, it raises a RuntimeException. print_command (bool): verbose (bool): is_interactive (bool): whether the command expects input from the user """ full_pod_name = get_pod_name(pod_name, deployment_target=deployment_target) if not full_pod_name: # assume it's already a full pod name full_pod_name = pod_name it_arg = "-it" if is_interactive else "" run("kubectl exec %(it_arg)s %(full_pod_name)s -- %(command)s" % locals(), errors_to_ignore=errors_to_ignore, print_command=print_command, verbose=verbose, is_interactive=is_interactive)
def deploy_secrets(settings, components=None): """Deploys or updates k8s secrets.""" if settings["ONLY_PUSH_TO_REGISTRY"]: return print_separator("secrets") create_namespace(settings) if not components: components = SECRETS.keys() # deploy secrets for secret_label in components: run("kubectl delete secret {}-secrets".format(secret_label), verbose=False, errors_to_ignore=["not found"]) for secret_label in components: secret_files = SECRETS.get(secret_label) if not secret_files: raise Exception('Invalid secret component {}'.format(secret_label)) secret_command = ['kubectl create secret generic {secret_label}-secrets'.format(secret_label=secret_label)] secret_command += [ '--from-file deploy/secrets/gcloud/{secret_label}/{file}'.format(secret_label=secret_label, file=file) for file in secret_files ] if secret_label == GCLOUD_CLIENT: secret_command.append('--from-file deploy/secrets/shared/gcloud/boto') run(" ".join(secret_command).format(deploy_to=settings['DEPLOY_TO']), errors_to_ignore=["already exists"])
def copy_files_to_or_from_pod(component, deployment_target, source_path, dest_path, direction=1): """Copy file(s) to or from the given component. Args: component (string): component label (eg. "postgres") deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" source_path (string): source file path. If copying files to the component, it should be a local path. Otherwise, it should be a file path inside the component pod. dest_path (string): destination file path. If copying files from the component, it should be a local path. Otherwise, it should be a file path inside the component pod. direction (int): If > 0 the file will be copied to the pod. If < 0, then it will be copied from the pod. """ full_pod_name = get_pod_name(component, deployment_target=deployment_target) if not full_pod_name: raise ValueError( "No '%(pod_name)s' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals()) if direction < 0: # copy from pod source_path = "%s:%s" % (full_pod_name, source_path) elif direction > 0: # copy to pod dest_path = "%s:%s" % (full_pod_name, dest_path) run("kubectl cp '%(source_path)s' '%(dest_path)s'" % locals())
def create_namespace(settings): run("kubectl create -f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/namespace.yaml" % settings, errors_to_ignore=["already exists"]) # switch kubectl to use the new namespace run("kubectl config set-context $(kubectl config current-context) --namespace=%(NAMESPACE)s" % settings)
def _set_elasticsearch_kubernetes_resources(): has_kube_resource = run('kubectl explain elasticsearch', errors_to_ignore=[ "server doesn't have a resource type", "couldn't find resource for" ]) if not has_kube_resource: run('kubectl apply -f deploy/kubernetes/elasticsearch/kubernetes-elasticsearch-all-in-one.yaml' )
def deploy_kube_scan(settings): print_separator("kube-scan") if settings["DELETE_BEFORE_DEPLOY"]: run("kubectl delete -f https://raw.githubusercontent.com/octarinesec/kube-scan/master/kube-scan.yaml") if settings["ONLY_PUSH_TO_REGISTRY"]: return run("kubectl apply -f https://raw.githubusercontent.com/octarinesec/kube-scan/master/kube-scan.yaml")
def troubleshoot_component(component, deployment_target): """Runs kubectl command to print detailed debug output for the given component. Args: component (string): component label (eg. "postgres") deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" """ pod_name = get_pod_name(component, deployment_target=deployment_target) run("kubectl get pods -o yaml %(pod_name)s" % locals(), verbose=True)
def deploy_external_connector(settings, connector_name): if connector_name not in ["elasticsearch"]: raise ValueError("Invalid connector name: %s" % connector_name) if settings["ONLY_PUSH_TO_REGISTRY"]: return print_separator("external-%s-connector" % connector_name) run(( "kubectl apply -f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/external-connectors/" % settings) + "external-%(connector_name)s.yaml" % locals())
def delete_pod(component_label, settings, custom_yaml_filename=None): deployment_target = settings["DEPLOY_TO"] yaml_filename = custom_yaml_filename or (component_label+".gcloud.yaml") if is_pod_running(component_label, deployment_target): run(" ".join([ "kubectl delete", "-f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/"+component_label+"/"+yaml_filename, ]) % settings, errors_to_ignore=["not found"]) logger.info("waiting for \"%s\" to exit Running status" % component_label) while is_pod_running(component_label, deployment_target): time.sleep(5)
def set_environment(deployment_target): """Configure the shell environment to point to the given deployment_target using 'gcloud config set-context' and other commands. Args: deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev", etc. """ settings = collections.OrderedDict() load_settings([ "deploy/kubernetes/shared-settings.yaml", "deploy/kubernetes/%(deployment_target)s-settings.yaml" % locals(), ], settings) if deployment_target.startswith("gcloud"): os.environ["KUBECONFIG"] = os.path.expanduser("~/.kube/config") run("gcloud config set core/project %(GCLOUD_PROJECT)s" % settings, print_command=True) run("gcloud config set compute/zone %(GCLOUD_ZONE)s" % settings, print_command=True) run("gcloud container clusters get-credentials --zone=%(GCLOUD_ZONE)s %(CLUSTER_NAME)s" % settings, print_command=True) else: raise ValueError("Unexpected deployment_target value: %s" % (deployment_target, )) run("kubectl config set-context $(kubectl config current-context) --namespace=%(NAMESPACE)s" % settings)
def delete_all(deployment_target): """Runs kubectl and gcloud commands to delete the given cluster and all objects in it. Args: deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" """ run('deploy/kubectl_helpers/utils/check_context.sh {}'.format( deployment_target.replace('gcloud-', ''))) settings = {} load_settings([ "deploy/kubernetes/shared-settings.yaml", "deploy/kubernetes/%(deployment_target)s-settings.yaml" % locals(), ], settings) run("gcloud container clusters delete --project %(GCLOUD_PROJECT)s --zone %(GCLOUD_ZONE)s --no-async %(CLUSTER_NAME)s" % settings, is_interactive=True) run('gcloud sql instances delete postgres-{}'.format( deployment_target.replace('gcloud-', ''))) for disk_label in [d.strip() for d in settings['DISKS'].split(',') if d]: for disk_name in get_disk_names(disk_label, settings): run('gcloud compute disks delete --zone {zone} {disk_name}'.format( zone=settings['GCLOUD_ZONE'], disk_name=disk_name), is_interactive=True)
def _get_resource_info( resource_type="pod", labels={}, json_path="{.items[].metadata.name}", errors_to_ignore=("array index out of bounds:",), verbose=False, ): """Runs 'kubectl get <resource_type> -l <label1=value1,label2=value2...> -o jsonpath=<json path>' and returns its output. Args: resource_type (string): "pod", "service", etc. labels (dict): (eg. {'name': 'phenotips'}) json_path (string): a json path query string (for example, "{.items[*].metadata.name}") errors_to_ignore (list): verbose (bool): Returns: (string) kubectl command output (eg. "postgres-410765475-1vtkn") or None if the kubectl command returned nothing """ l_arg = "-l {}".format(",".join(["%s=%s" % (key, value) for key, value in labels.items()])) if labels else "" output = run( "kubectl get %(resource_type)s %(l_arg)s -o jsonpath=%(json_path)s" % locals(), errors_to_ignore=errors_to_ignore, print_command=False, verbose=verbose, ) return output.strip('\n') if output is not None else None
def deploy_seqr(settings): print_separator("seqr") if settings["BUILD_DOCKER_IMAGES"]: seqr_git_hash = run("git log -1 --pretty=%h", errors_to_ignore=["Not a git repository"]) seqr_git_hash = ( ":" + seqr_git_hash.strip()) if seqr_git_hash is not None else "" docker_build("seqr", settings, [ "--build-arg SEQR_SERVICE_PORT=%s" % settings["SEQR_SERVICE_PORT"], "--build-arg SEQR_UI_DEV_PORT=%s" % settings["SEQR_UI_DEV_PORT"], "-f deploy/docker/seqr/Dockerfile", "-t %(DOCKER_IMAGE_NAME)s" + seqr_git_hash, ]) if settings["ONLY_PUSH_TO_REGISTRY"]: return restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP") reset_db = settings.get("RESET_DB") deployment_target = settings["DEPLOY_TO"] postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target) if settings["DELETE_BEFORE_DEPLOY"]: delete_pod("seqr", settings) elif reset_db or restore_seqr_db_from_backup: seqr_pod_name = get_pod_name('seqr', deployment_target=deployment_target) if seqr_pod_name: sleep_until_pod_is_running("seqr", deployment_target=deployment_target) run_in_pod(seqr_pod_name, "/usr/local/bin/stop_server.sh", verbose=True) if reset_db: _drop_seqr_db(postgres_pod_name) if restore_seqr_db_from_backup: _drop_seqr_db(postgres_pod_name) _restore_seqr_db_from_backup(postgres_pod_name, restore_seqr_db_from_backup) else: run_in_pod( postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'", errors_to_ignore=["already exists"], verbose=True, ) run_in_pod( postgres_pod_name, "psql -U postgres postgres -c 'create database reference_data_db'", errors_to_ignore=["already exists"], verbose=True, ) deploy_pod("seqr", settings, wait_until_pod_is_ready=True)
def docker_build(component_label, settings, custom_build_args=()): params = dict(settings) # make a copy before modifying params["COMPONENT_LABEL"] = component_label params[ "DOCKER_IMAGE_NAME"] = "%(DOCKER_IMAGE_PREFIX)s/%(COMPONENT_LABEL)s" % params docker_tags = set([ "", ":latest", ":%(TIMESTAMP)s" % settings, ]) if settings.get("DOCKER_IMAGE_TAG"): docker_tags.add(params["DOCKER_IMAGE_TAG"]) if component_label == 'elasticsearch' and settings.get( 'ELASTICSEARCH_VERSION'): docker_tags.add("%(DOCKER_IMAGE_TAG)s-%(ELASTICSEARCH_VERSION)s" % settings) if not settings["BUILD_DOCKER_IMAGES"]: logger.info( "Skipping docker build step. Use --build-docker-image to build a new image (and --force to build from the beginning)" ) else: docker_build_command = "" docker_build_command += "docker build deploy/docker/%(COMPONENT_LABEL)s/ " docker_build_command += (" ".join(custom_build_args) + " ") if settings["FORCE_BUILD_DOCKER_IMAGES"]: docker_build_command += "--no-cache " for tag in docker_tags: docker_image_name_with_tag = params["DOCKER_IMAGE_NAME"] + tag docker_build_command += "-t %(docker_image_name_with_tag)s " % locals( ) run(docker_build_command % params, verbose=True) if settings["PUSH_TO_REGISTRY"]: for tag in docker_tags: docker_image_name_with_tag = params["DOCKER_IMAGE_NAME"] + tag docker_push_command = "" docker_push_command += "docker push %(docker_image_name_with_tag)s" % locals( ) run(docker_push_command, verbose=True) logger.info( "==> Finished uploading image: %(docker_image_name_with_tag)s" % locals())
def _restore_seqr_db_from_backup(postgres_pod_name, seqrdb_backup, reference_data_backup=None): run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'", verbose=True) run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database reference_data_db'", verbose=True) run("kubectl cp '{backup}' {postgres_pod_name}:/root/$(basename {backup})".format( postgres_pod_name=postgres_pod_name, backup=seqrdb_backup), verbose=True) run_in_pod( postgres_pod_name, "/root/restore_database_backup.sh postgres seqrdb /root/$(basename {backup})".format( backup=seqrdb_backup), verbose=True) run_in_pod(postgres_pod_name, "rm /root/$(basename {backup})".format(backup=seqrdb_backup, verbose=True)) if reference_data_backup: run("kubectl cp '{backup}' {postgres_pod_name}:/root/$(basename {backup})".format( postgres_pod_name=postgres_pod_name, backup=reference_data_backup), verbose=True) run_in_pod( postgres_pod_name, "/root/restore_database_backup.sh postgres reference_data_db /root/$(basename {backup})".format( backup=reference_data_backup), verbose=True) run_in_pod(postgres_pod_name, "rm /root/$(basename {backup})".format(backup=reference_data_backup, verbose=True))
def deploy_pod(component_label, settings, wait_until_pod_is_running=True, wait_until_pod_is_ready=False): if settings["ONLY_PUSH_TO_REGISTRY"]: return if settings["DELETE_BEFORE_DEPLOY"]: delete_pod(component_label, settings) run(" ".join([ "kubectl apply", "-f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/"+component_label+"/"+component_label+".gcloud.yaml" ]) % settings) if wait_until_pod_is_running: sleep_until_pod_is_running(component_label, deployment_target=settings["DEPLOY_TO"]) if wait_until_pod_is_ready: sleep_until_pod_is_ready(component_label, deployment_target=settings["DEPLOY_TO"])
def deploy(deployment_target, components, output_dir=None, runtime_settings={}): """Deploy one or more components to the kubernetes cluster specified as the deployment_target. Args: deployment_target (string): value from DEPLOYMENT_ENVS - eg. "gcloud-dev" indentifying which cluster to deploy these components to components (list): The list of component names to deploy (eg. "postgres", "redis" - each string must be in constants.DEPLOYABLE_COMPONENTS). Order doesn't matter. output_dir (string): path of directory where to put deployment logs and rendered config files runtime_settings (dict): a dictionary of other key-value pairs that override settings file(s) values. """ if not components: raise ValueError("components list is empty") if components and "init-cluster" not in components: run('deploy/kubectl_helpers/utils/check_context.sh {}'.format( deployment_target.replace('gcloud-', ''))) settings = prepare_settings_for_deployment(deployment_target, output_dir, runtime_settings) # make sure namespace exists if "init-cluster" not in components and not runtime_settings.get( "ONLY_PUSH_TO_REGISTRY"): create_namespace(settings) if components[0] == 'secrets': deploy_secrets(settings, components=components[1:]) return # call deploy_* functions for each component in "components" list, in the order that these components are listed in DEPLOYABLE_COMPONENTS for component in DEPLOYABLE_COMPONENTS: if component in components: # only deploy requested components func_name = "deploy_" + component.replace("-", "_") f = globals().get(func_name) if f is not None: f(settings) else: raise ValueError( "'deploy_{}' function not found. Is '{}' a valid component name?" .format(func_name, component))
def deploy_elasticsearch(settings): print_separator("elasticsearch") docker_build("elasticsearch", settings, ["--build-arg ELASTICSEARCH_SERVICE_PORT=%s" % settings["ELASTICSEARCH_SERVICE_PORT"]]) if settings["ONLY_PUSH_TO_REGISTRY"]: return _set_elasticsearch_kubernetes_resources() # create persistent volumes pv_template_path = 'deploy/kubernetes/elasticsearch/persistent-volumes/es-data.yaml' disk_names = get_disk_names('es-data', settings) for disk_name in disk_names: volume_settings = {'DISK_NAME': disk_name} volume_settings.update(settings) _process_templates(volume_settings, [pv_template_path]) run('kubectl create -f {}/{}'.format(settings['DEPLOYMENT_TEMP_DIR'], pv_template_path), print_command=True, errors_to_ignore=['already exists']) deploy_pod("elasticsearch", settings, wait_until_pod_is_running=False) wait_for_not_resource( 'elasticsearch', resource_type='elasticsearch', json_path='{.items[0].status.phase}', invalid_status='Invalid', deployment_target=settings["DEPLOY_TO"], verbose_template='elasticsearch status') total_pods = 0 for num_pods in ['ES_DATA_NUM_PODS', 'ES_CLIENT_NUM_PODS', 'ES_MASTER_NUM_PODS', 'ES_LOADING_NUM_PODS']: total_pods += settings.get(num_pods, 0) for pod_number_i in range(total_pods): sleep_until_pod_is_running('elasticsearch', deployment_target=settings["DEPLOY_TO"], pod_number=pod_number_i) for pod_number_i in range(total_pods): sleep_until_pod_is_ready('elasticsearch', deployment_target=settings["DEPLOY_TO"], pod_number=pod_number_i) wait_for_resource( 'elasticsearch', resource_type='elasticsearch', json_path='{.items[0].status.phase}', expected_status='Ready', deployment_target=settings["DEPLOY_TO"], verbose_template='elasticsearch status') wait_for_resource( 'elasticsearch', resource_type='elasticsearch', json_path='{.items[0].status.health}', expected_status='green', deployment_target=settings["DEPLOY_TO"], verbose_template='elasticsearch health')
def create_vpc(gcloud_project, network_name): run( " ".join([ #"gcloud compute networks create seqr-project-custom-vpc --project=%(GCLOUD_PROJECT)s --mode=custom" "gcloud compute networks create %(network_name)s", "--project=%(gcloud_project)s", "--subnet-mode=auto" ]) % locals(), errors_to_ignore=["already exists"]) # add recommended firewall rules to enable ssh, etc. run(" ".join([ "gcloud compute firewall-rules create custom-vpc-allow-tcp-udp-icmp", "--project %(gcloud_project)s", "--network %(network_name)s", "--allow tcp,udp,icmp", "--source-ranges 10.0.0.0/8", ]) % locals(), errors_to_ignore=["already exists"]) run(" ".join([ "gcloud compute firewall-rules create custom-vpc-allow-ports", "--project %(gcloud_project)s", "--network %(network_name)s", "--allow tcp:22,tcp:3389,icmp", "--source-ranges 10.0.0.0/8", ]) % locals(), errors_to_ignore=["already exists"])
def deploy_init_cluster(settings): """Provisions a GKE cluster, persistent disks, and any other prerequisites for deployment.""" print_separator("init-cluster") # initialize the VM _init_cluster_gcloud(settings) node_name = get_node_name() if not node_name: raise Exception("Unable to retrieve node name. Was the cluster created successfully?") set_environment(settings["DEPLOY_TO"]) create_namespace(settings) # create priority classes - " Priority affects scheduling order of Pods and out-of-resource eviction ordering # on the Node.... A PriorityClass is a non-namespaced object .. The higher the value, the higher the priority." # (from https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) run("kubectl create priorityclass medium-priority --value=1000" % settings, errors_to_ignore=["already exists"]) run("kubectl create priorityclass high-priority --value=10000" % settings, errors_to_ignore=["already exists"]) # print cluster info run("kubectl cluster-info", verbose=True) # wait for the cluster to initialize for retry_i in range(1, 5): try: deploy_settings(settings) break except RuntimeError as e: logger.error(("Error when deploying config maps: %(e)s. This sometimes happens when cluster is " "initializing. Retrying...") % locals()) time.sleep(5)
def _init_gcloud_disks(settings): for disk_label in [d.strip() for d in settings['DISKS'].split(',') if d]: setting_prefix = disk_label.upper().replace('-', '_') disk_names = get_disk_names(disk_label, settings) snapshots = [ d.strip() for d in settings.get( '{}_SNAPSHOTS'.format(setting_prefix), '').split(',') if d ] if snapshots and len(snapshots) != len(disk_names): raise Exception( 'Invalid configuration for {}: {} disks to create and {} snapshots' .format(disk_label, len(disk_names), len(snapshots))) for i, disk_name in enumerate(disk_names): command = [ 'gcloud compute disks create', disk_name, '--zone', settings['GCLOUD_ZONE'], ] if settings.get('{}_DISK_TYPE'.format(setting_prefix)): command += [ '--type', settings['{}_DISK_TYPE'.format(setting_prefix)] ] if snapshots: command += ['--source-snapshot', snapshots[i]] else: command += [ '--size', str(settings['{}_DISK_SIZE'.format(setting_prefix)]) ] run(' '.join(command), verbose=True, errors_to_ignore=['lready exists'])
def deploy_nginx(settings): if settings["ONLY_PUSH_TO_REGISTRY"]: return print_separator("nginx") run("kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.41.2/deploy/static/provider/cloud/deploy.yaml" % locals()) if settings["DELETE_BEFORE_DEPLOY"]: run("kubectl delete -f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/nginx/nginx.yaml" % settings, errors_to_ignore=["not found"]) run("kubectl apply -f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/nginx/nginx.yaml" % settings)
def deploy_elasticsearch_sharded(settings, component): if settings["ONLY_PUSH_TO_REGISTRY"]: return print_separator(component) if component == "es-master": config_files = [ "%(DEPLOYMENT_TEMP_DIR)s/hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/es-discovery-svc.yaml", "%(DEPLOYMENT_TEMP_DIR)s/hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/es-master.yaml", ] elif component == "es-client": config_files = [ "%(DEPLOYMENT_TEMP_DIR)s/hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/es-svc.yaml", "%(DEPLOYMENT_TEMP_DIR)s/hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/es-client.yaml", ] elif component == "es-data": config_files = [ "%(DEPLOYMENT_TEMP_DIR)s/hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/es-data-svc.yaml", "%(DEPLOYMENT_TEMP_DIR)s/hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/es-data-stateful.yaml", ] elif component == "es-kibana": config_files = [ "%(DEPLOYMENT_TEMP_DIR)s/hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/es-kibana.yaml", ] elif component == "kibana": config_files = [ "%(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/kibana/kibana.%(DEPLOY_TO_PREFIX)s.yaml", ] else: raise ValueError("Unexpected component: " + component) if settings["DELETE_BEFORE_DEPLOY"]: for config_file in config_files: run("kubectl delete -f " + config_file % settings, errors_to_ignore=["not found"]) for config_file in config_files: run("kubectl apply -f " + config_file % settings) if component in ["es-client", "es-master", "es-data", "es-kibana"]: # wait until all replicas are running num_pods = int( settings.get(component.replace("-", "_").upper() + "_NUM_PODS", 1)) for pod_number_i in range(num_pods): sleep_until_pod_is_running(component, deployment_target=settings["DEPLOY_TO"], pod_number=pod_number_i) if component == "es-client": run("kubectl describe svc elasticsearch")
def deploy_linkerd(settings): print_separator('linkerd') version_match = run("linkerd version | awk '/Client/ {print $3}'") if version_match.strip() != settings["LINKERD_VERSION"]: raise Exception("Your locally installed linkerd version does not match %s. " "Download the correct version from https://github.com/linkerd/linkerd2/releases/tag/%s" % \ (settings['LINKERD_VERSION'], settings['LINKERD_VERSION'])) has_namespace = run('kubectl get namespace linkerd', errors_to_ignore=['namespaces "linkerd" not found']) if not has_namespace: run('linkerd install | kubectl apply -f -') run('linkerd check')
def deploy_seqr(settings): print_separator("seqr") if settings["BUILD_DOCKER_IMAGES"]: seqr_git_hash = run("git log -1 --pretty=%h", errors_to_ignore=["Not a git repository"]) seqr_git_hash = ( ":" + seqr_git_hash.strip()) if seqr_git_hash is not None else "" docker_build("seqr", settings, [ "--build-arg SEQR_SERVICE_PORT=%s" % settings["SEQR_SERVICE_PORT"], "-f deploy/docker/seqr/Dockerfile", "-t %(DOCKER_IMAGE_NAME)s" + seqr_git_hash, ]) if settings["ONLY_PUSH_TO_REGISTRY"]: return if settings["DELETE_BEFORE_DEPLOY"]: delete_pod("seqr", settings) deploy_pod("seqr", settings, wait_until_pod_is_ready=True)
def deploy_elasticsearch_snapshot_infra(settings): print_separator('elasticsearch snapshot infra') if settings['ES_CONFIGURE_SNAPSHOTS']: # create the bucket run("gsutil mb -p seqr-project -c STANDARD -l US-CENTRAL1 gs://%(ES_SNAPSHOTS_BUCKET)s" % settings, errors_to_ignore=["already exists"]) # create the IAM user run(" ".join([ "gcloud iam service-accounts create %(ES_SNAPSHOTS_ACCOUNT_NAME)s", "--display-name %(ES_SNAPSHOTS_ACCOUNT_NAME)s"]) % settings, errors_to_ignore="already exists within project projects/seqr-project") # grant storage admin permissions on the snapshot bucket run(" ".join([ "gsutil iam ch", "serviceAccount:%(ES_SNAPSHOTS_ACCOUNT_NAME)[email protected]:roles/storage.admin", "gs://%(ES_SNAPSHOTS_BUCKET)s"]) % settings)
def deploy_elasticsearch_snapshot_config(settings): print_separator('elasticsearch snapshot configuration') docker_build("curl", settings) if settings["ONLY_PUSH_TO_REGISTRY"]: return if settings['ES_CONFIGURE_SNAPSHOTS']: # run the k8s job to set up the repo run('kubectl apply -f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/elasticsearch/configure-snapshot-repo.yaml' % settings) wait_for_resource( 'configure-es-snapshot-repo', resource_type='job', json_path='{.items[0].status.conditions[0].type}', expected_status='Complete') # clean up the job after completion run('kubectl delete -f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/elasticsearch/configure-snapshot-repo.yaml' % settings) # Set up the monthly cron job run('kubectl apply -f %(DEPLOYMENT_TEMP_DIR)s/deploy/kubernetes/elasticsearch/snapshot-cronjob.yaml' % settings)
def deploy_settings(settings): """Deploy settings as a config map""" if settings["ONLY_PUSH_TO_REGISTRY"]: return # write out a ConfigMap file configmap_file_path = os.path.join(settings["DEPLOYMENT_TEMP_DIR"], "deploy/kubernetes/all-settings.properties") with open(configmap_file_path, "w") as f: for key, value in settings.items(): if value is None: continue f.write('%s=%s\n' % (key, value)) create_namespace(settings) run("kubectl delete configmap all-settings", errors_to_ignore=["not found"]) run("kubectl create configmap all-settings --from-file=%(configmap_file_path)s" % locals()) run("kubectl get configmaps all-settings -o yaml")
def delete_component(component, deployment_target=None): """Runs kubectl commands to delete any running deployment, service, or pod objects for the given component(s). Args: component (string): component to delete (eg. 'postgres' or 'nginx'). deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" """ if component == "cockpit": run("kubectl delete rc cockpit", errors_to_ignore=["not found"]) elif component == 'elasticsearch': run('kubectl delete elasticsearch elasticsearch', errors_to_ignore=['not found']) # Deleting a released persistent volume does not delete the data on the underlying disk wait_for_resource(component, '{.items[0].status.phase}', 'Released', deployment_target=deployment_target, resource_type='pv') pv = get_resource_name(component, resource_type='pv', deployment_target=deployment_target) while pv: run('kubectl delete pv {}'.format(pv)) pv = get_resource_name(component, resource_type='pv', deployment_target=deployment_target) elif component == 'kibana': run('kubectl delete kibana kibana', errors_to_ignore=['not found']) elif component == "nginx": raise ValueError("TODO: implement deleting nginx") run("kubectl delete deployments %(component)s" % locals(), errors_to_ignore=["not found"]) run("kubectl delete services %(component)s" % locals(), errors_to_ignore=["not found"]) pod_name = get_pod_name(component, deployment_target=deployment_target) if pod_name: run("kubectl delete pods %(pod_name)s" % locals(), errors_to_ignore=["not found"]) logger.info("waiting for \"%s\" to exit Running status" % component) while is_pod_running(component, deployment_target): time.sleep(5) # print services and pods status run("kubectl get services" % locals(), verbose=True) run("kubectl get pods" % locals(), verbose=True)
def show_dashboard(): """Opens the kubernetes dashboard in a new browser window.""" p = run_in_background('kubectl proxy') run('open http://localhost:8001/ui') p.wait()