def reset_database(database=[], deployment_target=None): """Runs kubectl commands to delete and reset the given database(s). Args: component (list): one more database labels - "seqrdb", "phenotipsdb", "mongodb" deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS. """ if "seqrdb" in database: postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target) if not postgres_pod_name: logger.error("postgres pod must be running") else: run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(), errors_to_ignore=["does not exist"]) run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals()) if "phenotipsdb" in database: postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target) if not postgres_pod_name: logger.error("postgres pod must be running") else: run_in_pod(postgres_pod_name, "psql -U xwiki postgres -c 'drop database xwiki'" % locals(), errors_to_ignore=["does not exist"]) run_in_pod(postgres_pod_name, "psql -U xwiki postgres -c 'create database xwiki'" % locals()) #run("kubectl exec %(postgres_pod_name)s -- psql -U postgres xwiki < data/init_phenotipsdb.sql" % locals()) if "mongodb" in database: mongo_pod_name = get_pod_name("mongo", deployment_target=deployment_target) if not mongo_pod_name: logger.error("mongo pod must be running") else: run_in_pod(mongo_pod_name, "mongo datastore --eval 'db.dropDatabase()'" % locals())
def deploy_seqr(settings): print_separator("seqr") docker_build( "seqr", settings, [ "--build-arg SEQR_SERVICE_PORT=%s" % settings["SEQR_SERVICE_PORT"], "--build-arg SEQR_UI_DEV_PORT=%s" % settings["SEQR_UI_DEV_PORT"], "-f deploy/docker/%(COMPONENT_LABEL)s/%(DEPLOY_TO_PREFIX)s/Dockerfile" ], ) restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP") reset_db = settings.get("RESET_DB") deployment_target = settings["DEPLOY_TO"] postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target) if settings["DELETE_BEFORE_DEPLOY"]: delete_pod("seqr", settings) elif reset_db or restore_seqr_db_from_backup: seqr_pod_name = get_pod_name('seqr', deployment_target=deployment_target) if seqr_pod_name: _wait_until_pod_is_running("seqr", deployment_target=deployment_target) run_in_pod(seqr_pod_name, "/usr/local/bin/stop_server.sh" % locals(), verbose=True) if reset_db: run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(), errors_to_ignore=["does not exist"], verbose=True, ) if restore_seqr_db_from_backup: run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(), errors_to_ignore=["does not exist"], verbose=True, ) run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals(), verbose=True) run("kubectl cp '%(restore_seqr_db_from_backup)s' %(postgres_pod_name)s:/root/$(basename %(restore_seqr_db_from_backup)s)" % locals(), verbose=True) run_in_pod(postgres_pod_name, "/root/restore_database_backup.sh postgres seqrdb /root/$(basename %(restore_seqr_db_from_backup)s)" % locals(), verbose=True) run_in_pod(postgres_pod_name, "rm /root/$(basename %(restore_seqr_db_from_backup)s)" % locals(), verbose=True) else: run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals(), errors_to_ignore=["already exists"], verbose=True, ) _deploy_pod("seqr", settings, wait_until_pod_is_ready=True)
def load_example_project(deployment_target, genome_version="37"): """Load example project Args: genome_version (string): reference genome version - either "37" or "38" """ check_kubernetes_context(deployment_target) pod_name = get_pod_name('seqr', deployment_target=deployment_target) if not pod_name: raise ValueError("No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals()) if genome_version == "37": vcf_filename = "1kg.vep.vcf.gz" elif genome_version == "38": vcf_filename = "1kg.liftover.GRCh38.vep.vcf.gz" else: raise ValueError("Unexpected genome_version: %s" % (genome_version,)) project_id = "1kg" vcf = "https://storage.googleapis.com/seqr-public/test-projects/1kg-exomes/%(vcf_filename)s" % locals() ped = "https://storage.googleapis.com/seqr-public/test-projects/1kg-exomes/1kg.ped" load_project(deployment_target, project_id=project_id, genome_version=genome_version, vcf=vcf, ped=ped)
def load_allele_frequencies(deployment_target, genome_version="37"): """Load ExAC and 1kg allele frequency datasets. These are larger and take longer to load than other reference data Args: deployment_target (string): "minikube", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS. genome_version (string): reference genome version - either "37" or "38" """ check_kubernetes_context(deployment_target) pod_name = get_pod_name('seqr', deployment_target=deployment_target) if not pod_name: raise ValueError( "No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals()) run_in_pod( pod_name, "wget -N http://seqr.broadinstitute.org/static/bundle/ExAC.r0.3.sites.vep.popmax.clinvar.vcf.gz -P /data/reference_data/" % locals(), verbose=True) run_in_pod( pod_name, "wget -N http://seqr.broadinstitute.org/static/bundle/ALL.wgs.phase3_shapeit2_mvncall_integrated_v5a.20130502.sites.decomposed.with_popmax.vcf.gz -P /data/reference_data/" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py load_reference" % locals(), verbose=True)
def delete_component(component, deployment_target=None): """Runs kubectl commands to delete any running deployment, service, or pod objects for the given component(s). Args: component (string): component to delete (eg. 'phenotips' or 'nginx'). deployment_target (string): "minikube", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS. """ if component == "cockpit": run("kubectl delete rc cockpit", errors_to_ignore=["not found"]) elif component == "es-data": run("kubectl delete StatefulSet es-data", errors_to_ignore=["not found"]) elif component == "nginx": run("kubectl delete rc nginx-ingress-rc", errors_to_ignore=["not found"]) run("kubectl delete deployments %(component)s" % locals(), errors_to_ignore=["not found"]) run("kubectl delete services %(component)s" % locals(), errors_to_ignore=["not found"]) pod_name = get_pod_name(component, deployment_target=deployment_target) if pod_name: run("kubectl delete pods %(pod_name)s" % locals(), errors_to_ignore=["not found"]) run("kubectl get services" % locals()) run("kubectl get pods" % locals())
def troubleshoot_component(component, deployment_target): """Runs kubectl command to print detailed debug output for the given component. Args: component (string): component label (eg. "postgres") deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS. """ pod_name = get_pod_name(component, deployment_target=deployment_target) run("kubectl get pods -o yaml %(pod_name)s" % locals(), verbose=True)
def port_forward(component_port_pairs=[], deployment_target=None, wait=True, open_browser=False, use_kubectl_proxy=False): """Executes kubectl command to forward traffic between localhost and the given pod. While this is running, connecting to localhost:<port> will be the same as connecting to that port from the pod's internal network. Args: component_port_pairs (list): 2-tuple(s) containing keyword to use for looking up a kubernetes pod, along with the port to forward to that pod (eg. ('mongo', 27017), or ('phenotips', 8080)) deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS. wait (bool): Whether to block indefinitely as long as the forwarding process is running. open_browser (bool): If component_port_pairs includes components that have an http server (eg. "seqr" or "phenotips"), then open a web browser window to the forwarded port. use_kubectl_proxy (bool): Whether to use kubectl proxy instead of kubectl port-forward (see https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#manually-constructing-apiserver-proxy-urls) Returns: (list): Popen process objects for the kubectl port-forward processes. """ procs = [] for component_label, port in component_port_pairs: while get_pod_status(component_label, deployment_target) != "Running": time.sleep(5) logger.info("Forwarding port %s for %s" % (port, component_label)) pod_name = get_pod_name(component_label, deployment_target=deployment_target) if use_kubectl_proxy: command = "kubectl proxy --port 8001" else: command = "kubectl port-forward %(pod_name)s %(port)s" % locals() p = run_in_background(command) if open_browser and component_label in COMPONENTS_TO_OPEN_IN_BROWSER: if use_kubectl_proxy: url = "http://localhost:8001/api/v1/namespaces/default/services/%(component_label)s:%(port)s/proxy/" % locals( ) else: url = "http://localhost:%s" % port os.system("open " + url) procs.append(p) if wait: wait_for(procs) return procs
def load_reference_data(deployment_target, genome_version="37"): """Load reference data Args: genome_version (string): reference genome version - either "37" or "38" """ check_kubernetes_context(deployment_target) pod_name = get_pod_name('seqr', deployment_target=deployment_target) if not pod_name: raise ValueError( "No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals()) run_in_pod(pod_name, "mkdir -p /data/reference_data/" % locals()) run_in_pod( pod_name, "wget -N https://storage.googleapis.com/seqr-public/reference-data/seqr-resource-bundle.GRCh%(genome_version)s.tar.gz -P /data/reference_data/" % locals(), verbose=True) run_in_pod( pod_name, "tar -xzf /data/reference_data/seqr-resource-bundle.GRCh%(genome_version)s.tar.gz --directory /data/reference_data/" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py load_resources" % locals(), verbose=True) run_in_pod( pod_name, "python2.7 -u manage.py update_gencode /data/reference_data/gencode.v19.annotation.gtf.gz" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py update_human_phenotype_ontology" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py update_omim" % locals(), verbose=True) run_in_pod(pod_name, "/usr/local/bin/restart_server.sh" % locals(), verbose=True)
def print_log(components, deployment_target, enable_stream_log, wait=True): """Executes kubernetes command to print logs for the given pod. Args: components (list): one or more kubernetes pod labels (eg. 'phenotips' or 'nginx'). If more than one is specified, logs will be printed from all components in parallel. deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS. enable_stream_log (bool): whether to continuously stream the log instead of just printing the log up to now. wait (bool): If False, this method will return without waiting for the log streaming process to finish printing all logs. Returns: (list): Popen process objects for the kubectl port-forward processes. """ stream_arg = "-f" if enable_stream_log else "" procs = [] for component_label in components: while get_pod_status(component_label, deployment_target) != "Running": time.sleep(5) pod_name = get_pod_name(component_label, deployment_target=deployment_target) p = run_in_background("kubectl logs %(stream_arg)s %(pod_name)s" % locals()) def print_command_log(): for line in iter(p.stdout.readline, ''): logger.info(line.strip('\n')) t = Thread(target=print_command_log) t.start() procs.append(p) if wait: wait_for(procs) return procs
def load_project(deployment_target, project_id="1kg", genome_version="37", vcf=None, ped=None): """Load example project Args: project_id (string): project id genome_version (string): reference genome version - either "37" or "38" vcf (string): VCF path ped (string): PED path """ check_kubernetes_context(deployment_target) pod_name = get_pod_name('seqr', deployment_target=deployment_target) if not pod_name: raise ValueError("No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals()) if not project_id: raise ValueError("project_id not specified") if not vcf: raise ValueError("vcf not specified") if not ped: raise ValueError("ped not specified") vcf_filename = os.path.basename(vcf) ped_filename = os.path.basename(ped) run_in_pod(pod_name, "wget -N %(vcf)s" % locals()) run_in_pod(pod_name, "wget -N %(ped)s" % locals()) run_in_pod(pod_name, "python2.7 -u manage.py add_project '%(project_id)s' '%(project_id)s'" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py add_individuals_to_project '%(project_id)s' --ped '%(ped_filename)s'" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py add_vcf_to_project --clear '%(project_id)s' '%(vcf_filename)s'" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py add_project_to_phenotips '%(project_id)s' '%(project_id)s'" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py add_individuals_to_phenotips '%(project_id)s' --ped '%(ped_filename)s'" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py generate_pedigree_images -f '%(project_id)s'" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py add_default_tags '%(project_id)s'" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py load_project '%(project_id)s'" % locals(), verbose=True) run_in_pod(pod_name, "python2.7 -u manage.py load_project_datastore '%(project_id)s'" % locals(), verbose=True)
def deploy_phenotips(settings): print_separator("phenotips") phenotips_service_port = settings["PHENOTIPS_SERVICE_PORT"] restore_phenotips_db_from_backup = settings.get( "RESTORE_PHENOTIPS_DB_FROM_BACKUP") reset_db = settings.get("RESET_DB") deployment_target = settings["DEPLOY_TO"] if reset_db or restore_phenotips_db_from_backup: delete_pod("phenotips", settings) run_in_pod( "postgres", "psql -U postgres postgres -c 'drop database xwiki'" % locals(), verbose=True, errors_to_ignore=["does not exist"], deployment_target=deployment_target, ) elif settings["DELETE_BEFORE_DEPLOY"]: delete_pod("phenotips", settings) run_in_pod( "postgres", "psql -U postgres postgres -c \"create role xwiki with CREATEDB LOGIN PASSWORD 'xwiki'\"" % locals(), verbose=True, errors_to_ignore=["already exists"], deployment_target=deployment_target, ) run_in_pod( "postgres", "psql -U xwiki postgres -c 'create database xwiki'" % locals(), verbose=True, errors_to_ignore=["already exists"], deployment_target=deployment_target, ) run_in_pod( "postgres", "psql -U postgres postgres -c 'grant all privileges on database xwiki to xwiki'" % locals(), ) docker_build( "phenotips", settings, ["--build-arg PHENOTIPS_SERVICE_PORT=%s" % phenotips_service_port], ) _deploy_pod("phenotips", settings, wait_until_pod_is_ready=True) for i in range(0, 3): # opening the PhenoTips website for the 1st time triggers a final set of initialization # steps which take ~ 1 minute, so run wget to trigger this try: run_in_pod( "phenotips", #command="wget http://localhost:%(phenotips_service_port)s -O test.html" % locals(), command= "curl --verbose -L -u Admin:admin http://localhost:%(phenotips_service_port)s -o test.html" % locals(), verbose=True) except Exception as e: logger.error(str(e)) if i < 2: logger.info("Waiting for phenotips to start up...") time.sleep(10) if restore_phenotips_db_from_backup: delete_pod("phenotips", settings) postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target) run("kubectl cp '%(restore_phenotips_db_from_backup)s' %(postgres_pod_name)s:/root/$(basename %(restore_phenotips_db_from_backup)s)" % locals(), verbose=True) run_in_pod( "postgres", "/root/restore_database_backup.sh xwiki xwiki /root/$(basename %(restore_phenotips_db_from_backup)s)" % locals(), deployment_target=deployment_target, verbose=True) run_in_pod( "postgres", "rm /root/$(basename %(restore_phenotips_db_from_backup)s)" % locals(), deployment_target=deployment_target, verbose=True) _deploy_pod("phenotips", settings, wait_until_pod_is_ready=True)