def deploy_seqr(settings): print_separator("seqr") if settings["BUILD_DOCKER_IMAGES"]: seqr_git_hash = run("git log -1 --pretty=%h", errors_to_ignore=["Not a git repository"]) seqr_git_hash = ( ":" + seqr_git_hash.strip()) if seqr_git_hash is not None else "" docker_build("seqr", settings, [ "--build-arg SEQR_SERVICE_PORT=%s" % settings["SEQR_SERVICE_PORT"], "--build-arg SEQR_UI_DEV_PORT=%s" % settings["SEQR_UI_DEV_PORT"], "-f deploy/docker/seqr/Dockerfile", "-t %(DOCKER_IMAGE_NAME)s" + seqr_git_hash, ]) if settings["ONLY_PUSH_TO_REGISTRY"]: return restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP") reset_db = settings.get("RESET_DB") deployment_target = settings["DEPLOY_TO"] postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target) if settings["DELETE_BEFORE_DEPLOY"]: delete_pod("seqr", settings) elif reset_db or restore_seqr_db_from_backup: seqr_pod_name = get_pod_name('seqr', deployment_target=deployment_target) if seqr_pod_name: sleep_until_pod_is_running("seqr", deployment_target=deployment_target) run_in_pod(seqr_pod_name, "/usr/local/bin/stop_server.sh", verbose=True) if reset_db: _drop_seqr_db(postgres_pod_name) if restore_seqr_db_from_backup: _drop_seqr_db(postgres_pod_name) _restore_seqr_db_from_backup(postgres_pod_name, restore_seqr_db_from_backup) else: run_in_pod( postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'", errors_to_ignore=["already exists"], verbose=True, ) run_in_pod( postgres_pod_name, "psql -U postgres postgres -c 'create database reference_data_db'", errors_to_ignore=["already exists"], verbose=True, ) deploy_pod("seqr", settings, wait_until_pod_is_ready=True)
def deploy_postgres(settings): print_separator("postgres") docker_build("postgres", settings) restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP") reset_db = settings.get("RESET_DB") if reset_db or restore_seqr_db_from_backup: # Since pgdata is stored on a persistent volume, redeploying does not get rid of it. If any existing pgdata is # present, even if the databases are empty, postgres will not fully re-initialize the database. This is # good if you want to keep the data across a deployment, but problematic if you actually need to rest and # reinitialize the db. Therefore, when the database needs to be fully reinitialized, delete pgdata run_in_pod(get_pod_name("postgres", deployment_target=settings["DEPLOY_TO"]), "rm -rf /var/lib/postgresql/data/pgdata", verbose=True) deploy_pod("postgres", settings, wait_until_pod_is_ready=True) if restore_seqr_db_from_backup: postgres_pod_name = get_pod_name( "postgres", deployment_target=settings["DEPLOY_TO"]) _restore_seqr_db_from_backup( postgres_pod_name, restore_seqr_db_from_backup, settings.get("RESTORE_REFERENCE_DB_FROM_BACKUP"))
def copy_files_to_or_from_pod(component, deployment_target, source_path, dest_path, direction=1): """Copy file(s) to or from the given component. Args: component (string): component label (eg. "postgres") deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" source_path (string): source file path. If copying files to the component, it should be a local path. Otherwise, it should be a file path inside the component pod. dest_path (string): destination file path. If copying files from the component, it should be a local path. Otherwise, it should be a file path inside the component pod. direction (int): If > 0 the file will be copied to the pod. If < 0, then it will be copied from the pod. """ full_pod_name = get_pod_name(component, deployment_target=deployment_target) if not full_pod_name: raise ValueError( "No '%(pod_name)s' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals()) if direction < 0: # copy from pod source_path = "%s:%s" % (full_pod_name, source_path) elif direction > 0: # copy to pod dest_path = "%s:%s" % (full_pod_name, dest_path) run("kubectl cp '%(source_path)s' '%(dest_path)s'" % locals())
def delete_component(component, deployment_target=None): """Runs kubectl commands to delete any running deployment, service, or pod objects for the given component(s). Args: component (string): component to delete (eg. 'postgres' or 'nginx'). deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" """ if component == "cockpit": run("kubectl delete rc cockpit", errors_to_ignore=["not found"]) elif component == "es-data": run("kubectl delete StatefulSet es-data", errors_to_ignore=["not found"]) elif component == "nginx": raise ValueError("TODO: implement deleting nginx") run("kubectl delete deployments %(component)s" % locals(), errors_to_ignore=["not found"]) run("kubectl delete services %(component)s" % locals(), errors_to_ignore=["not found"]) pod_name = get_pod_name(component, deployment_target=deployment_target) if pod_name: run("kubectl delete pods %(pod_name)s" % locals(), errors_to_ignore=["not found"]) logger.info("waiting for \"%s\" to exit Running status" % component) while is_pod_running(component, deployment_target): time.sleep(5) # print services and pods status run("kubectl get services" % locals(), verbose=True) run("kubectl get pods" % locals(), verbose=True)
def troubleshoot_component(component, deployment_target): """Runs kubectl command to print detailed debug output for the given component. Args: component (string): component label (eg. "postgres") deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" """ pod_name = get_pod_name(component, deployment_target=deployment_target) run("kubectl get pods -o yaml %(pod_name)s" % locals(), verbose=True)
def redeploy_seqr(deployment_target): print_separator('re-deploying seqr') seqr_pod_name = get_pod_name('seqr', deployment_target=deployment_target) if not seqr_pod_name: raise ValueError('No seqr pod found, unable to re-deploy') sleep_until_pod_is_running('seqr', deployment_target=deployment_target) run_in_pod(seqr_pod_name, 'git pull', verbose=True) run_in_pod(seqr_pod_name, './manage.py migrate', verbose=True) run_in_pod(seqr_pod_name, '/usr/local/bin/restart_server.sh')
def print_log(components, deployment_target, enable_stream_log, previous=False, wait=True): """Executes kubernetes command to print logs for the given pod. Args: components (list): one or more kubernetes pod labels (eg. 'postgres' or 'nginx'). If more than one is specified, logs will be printed from all components in parallel. deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev", etc. enable_stream_log (bool): whether to continuously stream the log instead of just printing the log up to now. previous (bool): Prints logs from a previous instance of the container. This is useful for debugging pods that don't start or immediately enter crash-loop. wait (bool): If False, this method will return without waiting for the log streaming process to finish printing all logs. Returns: (list): Popen process objects for the kubectl port-forward processes. """ stream_arg = "-f" if enable_stream_log else "" previous_flag = "--previous" if previous else "" procs = [] for component_label in components: if component_label == "kube-scan": continue # See https://github.com/octarinesec/kube-scan for how to connect to the kube-scan pod. if not previous: wait_until_pod_is_running(component_label, deployment_target) pod_name = get_pod_name(component_label, deployment_target=deployment_target) p = run_in_background( "kubectl logs %(stream_arg)s %(previous_flag)s %(pod_name)s --all-containers" % locals(), print_command=True) def print_command_log(): for line in iter(p.stdout.readline, ''): logger.info(line.strip('\n')) t = Thread(target=print_command_log) t.start() procs.append(p) if wait: wait_for(procs) return procs
def reset_database(database=[], deployment_target=None): """Runs kubectl commands to delete and reset the given database(s). Args: component (list): one more database labels - "seqrdb" deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" """ if "seqrdb" in database: postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target) if not postgres_pod_name: logger.error("postgres pod must be running") else: run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(), errors_to_ignore=["does not exist"]) run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals())
def port_forward(component_port_pairs=[], deployment_target=None, wait=True, open_browser=False, use_kubectl_proxy=False): """Executes kubectl command to set up port forwarding between localhost and the given pod. While this is running, connecting to localhost:<port> will be the same as connecting to that port from the pod's internal network. Args: component_port_pairs (list): 2-tuple(s) containing keyword to use for looking up a kubernetes pod, along with the port to forward to that pod (eg. ('postgres', 5432)) deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" wait (bool): Whether to block indefinitely as long as the forwarding process is running. open_browser (bool): If component_port_pairs includes components that have an http server (eg. "seqr" or "postgres"), then open a web browser window to the forwarded port. use_kubectl_proxy (bool): Whether to use kubectl proxy instead of kubectl port-forward (see https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#manually-constructing-apiserver-proxy-urls) Returns: (list): Popen process objects for the kubectl port-forward processes. """ procs = [] for component_label, port in component_port_pairs: if component_label == "kube-scan": continue # See https://github.com/octarinesec/kube-scan for how to connect to the kube-scan pod. wait_until_pod_is_running(component_label, deployment_target) logger.info("Forwarding port %s for %s" % (port, component_label)) pod_name = get_pod_name(component_label, deployment_target=deployment_target) if use_kubectl_proxy: command = "kubectl proxy --port 8001" else: command = "kubectl port-forward %(pod_name)s %(port)s" % locals() p = run_in_background(command) if open_browser and component_label in COMPONENTS_TO_OPEN_IN_BROWSER: if use_kubectl_proxy: url = "http://localhost:8001/api/v1/namespaces/default/services/%(component_label)s:%(port)s/proxy/" % locals() else: url = "http://localhost:%s" % port time.sleep(3) os.system("open " + url) procs.append(p) if wait: wait_for(procs) return procs
def delete_component(component, deployment_target=None): """Runs kubectl commands to delete any running deployment, service, or pod objects for the given component(s). Args: component (string): component to delete (eg. 'postgres' or 'nginx'). deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev" """ if component == "cockpit": run("kubectl delete rc cockpit", errors_to_ignore=["not found"]) elif component == 'elasticsearch': run('kubectl delete elasticsearch elasticsearch', errors_to_ignore=['not found']) # Deleting a released persistent volume does not delete the data on the underlying disk wait_for_resource(component, '{.items[0].status.phase}', 'Released', deployment_target=deployment_target, resource_type='pv') pv = get_resource_name(component, resource_type='pv', deployment_target=deployment_target) while pv: run('kubectl delete pv {}'.format(pv)) pv = get_resource_name(component, resource_type='pv', deployment_target=deployment_target) elif component == 'kibana': run('kubectl delete kibana kibana', errors_to_ignore=['not found']) elif component == "nginx": raise ValueError("TODO: implement deleting nginx") run("kubectl delete deployments %(component)s" % locals(), errors_to_ignore=["not found"]) run("kubectl delete services %(component)s" % locals(), errors_to_ignore=["not found"]) pod_name = get_pod_name(component, deployment_target=deployment_target) if pod_name: run("kubectl delete pods %(pod_name)s" % locals(), errors_to_ignore=["not found"]) logger.info("waiting for \"%s\" to exit Running status" % component) while is_pod_running(component, deployment_target): time.sleep(5) # print services and pods status run("kubectl get services" % locals(), verbose=True) run("kubectl get pods" % locals(), verbose=True)