def reset_database(database=[], deployment_target=None):
"""Runs kubectl commands to delete and reset the given database(s).
Args:
component (list): one more database labels - "seqrdb", "phenotipsdb", "mongodb"
deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
"""
if "seqrdb" in database:
postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target)
if not postgres_pod_name:
logger.error("postgres pod must be running")
else:
run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(), errors_to_ignore=["does not exist"])
run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals())
if "phenotipsdb" in database:
postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target)
if not postgres_pod_name:
logger.error("postgres pod must be running")
else:
run_in_pod(postgres_pod_name, "psql -U xwiki postgres -c 'drop database xwiki'" % locals(), errors_to_ignore=["does not exist"])
run_in_pod(postgres_pod_name, "psql -U xwiki postgres -c 'create database xwiki'" % locals())
#run("kubectl exec %(postgres_pod_name)s -- psql -U postgres xwiki < data/init_phenotipsdb.sql" % locals())
if "mongodb" in database:
mongo_pod_name = get_pod_name("mongo", deployment_target=deployment_target)
if not mongo_pod_name:
logger.error("mongo pod must be running")
else:
run_in_pod(mongo_pod_name, "mongo datastore --eval 'db.dropDatabase()'" % locals())
def deploy_seqr(settings):
print_separator("seqr")
docker_build(
"seqr",
settings,
[
"--build-arg SEQR_SERVICE_PORT=%s" % settings["SEQR_SERVICE_PORT"],
"--build-arg SEQR_UI_DEV_PORT=%s" % settings["SEQR_UI_DEV_PORT"],
"-f deploy/docker/%(COMPONENT_LABEL)s/%(DEPLOY_TO_PREFIX)s/Dockerfile"
],
)
restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP")
reset_db = settings.get("RESET_DB")
deployment_target = settings["DEPLOY_TO"]
postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target)
if settings["DELETE_BEFORE_DEPLOY"]:
delete_pod("seqr", settings)
elif reset_db or restore_seqr_db_from_backup:
seqr_pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if seqr_pod_name:
_wait_until_pod_is_running("seqr", deployment_target=deployment_target)
run_in_pod(seqr_pod_name, "/usr/local/bin/stop_server.sh" % locals(), verbose=True)
if reset_db:
run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(),
errors_to_ignore=["does not exist"],
verbose=True,
)
if restore_seqr_db_from_backup:
run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(),
errors_to_ignore=["does not exist"],
verbose=True,
)
run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals(), verbose=True)
run("kubectl cp '%(restore_seqr_db_from_backup)s' %(postgres_pod_name)s:/root/$(basename %(restore_seqr_db_from_backup)s)" % locals(), verbose=True)
run_in_pod(postgres_pod_name, "/root/restore_database_backup.sh postgres seqrdb /root/$(basename %(restore_seqr_db_from_backup)s)" % locals(), verbose=True)
run_in_pod(postgres_pod_name, "rm /root/$(basename %(restore_seqr_db_from_backup)s)" % locals(), verbose=True)
else:
run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals(),
errors_to_ignore=["already exists"],
verbose=True,
)
_deploy_pod("seqr", settings, wait_until_pod_is_ready=True)
def load_example_project(deployment_target, genome_version="37"):
"""Load example project
Args:
genome_version (string): reference genome version - either "37" or "38"
"""
check_kubernetes_context(deployment_target)
pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not pod_name:
raise ValueError("No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals())
if genome_version == "37":
vcf_filename = "1kg.vep.vcf.gz"
elif genome_version == "38":
vcf_filename = "1kg.liftover.GRCh38.vep.vcf.gz"
else:
raise ValueError("Unexpected genome_version: %s" % (genome_version,))
project_id = "1kg"
vcf = "https://storage.googleapis.com/seqr-public/test-projects/1kg-exomes/%(vcf_filename)s" % locals()
ped = "https://storage.googleapis.com/seqr-public/test-projects/1kg-exomes/1kg.ped"
load_project(deployment_target, project_id=project_id, genome_version=genome_version, vcf=vcf, ped=ped)
def load_allele_frequencies(deployment_target, genome_version="37"):
"""Load ExAC and 1kg allele frequency datasets. These are larger and take longer to load than other reference data
Args:
deployment_target (string): "minikube", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
genome_version (string): reference genome version - either "37" or "38"
"""
check_kubernetes_context(deployment_target)
pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not pod_name:
raise ValueError(
"No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?"
% locals())
run_in_pod(
pod_name,
"wget -N http://seqr.broadinstitute.org/static/bundle/ExAC.r0.3.sites.vep.popmax.clinvar.vcf.gz -P /data/reference_data/"
% locals(),
verbose=True)
run_in_pod(
pod_name,
"wget -N http://seqr.broadinstitute.org/static/bundle/ALL.wgs.phase3_shapeit2_mvncall_integrated_v5a.20130502.sites.decomposed.with_popmax.vcf.gz -P /data/reference_data/"
% locals(),
verbose=True)
run_in_pod(pod_name,
"python2.7 -u manage.py load_reference" % locals(),
verbose=True)
def delete_component(component, deployment_target=None):
"""Runs kubectl commands to delete any running deployment, service, or pod objects for the
given component(s).
Args:
component (string): component to delete (eg. 'phenotips' or 'nginx').
deployment_target (string): "minikube", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
"""
if component == "cockpit":
run("kubectl delete rc cockpit", errors_to_ignore=["not found"])
elif component == "es-data":
run("kubectl delete StatefulSet es-data",
errors_to_ignore=["not found"])
elif component == "nginx":
run("kubectl delete rc nginx-ingress-rc",
errors_to_ignore=["not found"])
run("kubectl delete deployments %(component)s" % locals(),
errors_to_ignore=["not found"])
run("kubectl delete services %(component)s" % locals(),
errors_to_ignore=["not found"])
pod_name = get_pod_name(component, deployment_target=deployment_target)
if pod_name:
run("kubectl delete pods %(pod_name)s" % locals(),
errors_to_ignore=["not found"])
run("kubectl get services" % locals())
run("kubectl get pods" % locals())
def troubleshoot_component(component, deployment_target):
"""Runs kubectl command to print detailed debug output for the given component.
Args:
component (string): component label (eg. "postgres")
deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
"""
pod_name = get_pod_name(component, deployment_target=deployment_target)
run("kubectl get pods -o yaml %(pod_name)s" % locals(), verbose=True)
def port_forward(component_port_pairs=[],
deployment_target=None,
wait=True,
open_browser=False,
use_kubectl_proxy=False):
"""Executes kubectl command to forward traffic between localhost and the given pod.
While this is running, connecting to localhost:<port> will be the same as connecting to that port
from the pod's internal network.
Args:
component_port_pairs (list): 2-tuple(s) containing keyword to use for looking up a kubernetes
pod, along with the port to forward to that pod (eg. ('mongo', 27017), or ('phenotips', 8080))
deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
wait (bool): Whether to block indefinitely as long as the forwarding process is running.
open_browser (bool): If component_port_pairs includes components that have an http server
(eg. "seqr" or "phenotips"), then open a web browser window to the forwarded port.
use_kubectl_proxy (bool): Whether to use kubectl proxy instead of kubectl port-forward
(see https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#manually-constructing-apiserver-proxy-urls)
Returns:
(list): Popen process objects for the kubectl port-forward processes.
"""
procs = []
for component_label, port in component_port_pairs:
while get_pod_status(component_label, deployment_target) != "Running":
time.sleep(5)
logger.info("Forwarding port %s for %s" % (port, component_label))
pod_name = get_pod_name(component_label,
deployment_target=deployment_target)
if use_kubectl_proxy:
command = "kubectl proxy --port 8001"
else:
command = "kubectl port-forward %(pod_name)s %(port)s" % locals()
p = run_in_background(command)
if open_browser and component_label in COMPONENTS_TO_OPEN_IN_BROWSER:
if use_kubectl_proxy:
url = "http://localhost:8001/api/v1/namespaces/default/services/%(component_label)s:%(port)s/proxy/" % locals(
)
else:
url = "http://localhost:%s" % port
os.system("open " + url)
procs.append(p)
if wait:
wait_for(procs)
return procs
def load_reference_data(deployment_target, genome_version="37"):
"""Load reference data
Args:
genome_version (string): reference genome version - either "37" or "38"
"""
check_kubernetes_context(deployment_target)
pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not pod_name:
raise ValueError(
"No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?"
% locals())
run_in_pod(pod_name, "mkdir -p /data/reference_data/" % locals())
run_in_pod(
pod_name,
"wget -N https://storage.googleapis.com/seqr-public/reference-data/seqr-resource-bundle.GRCh%(genome_version)s.tar.gz -P /data/reference_data/"
% locals(),
verbose=True)
run_in_pod(
pod_name,
"tar -xzf /data/reference_data/seqr-resource-bundle.GRCh%(genome_version)s.tar.gz --directory /data/reference_data/"
% locals(),
verbose=True)
run_in_pod(pod_name,
"python2.7 -u manage.py load_resources" % locals(),
verbose=True)
run_in_pod(
pod_name,
"python2.7 -u manage.py update_gencode /data/reference_data/gencode.v19.annotation.gtf.gz"
% locals(),
verbose=True)
run_in_pod(pod_name,
"python2.7 -u manage.py update_human_phenotype_ontology" %
locals(),
verbose=True)
run_in_pod(pod_name,
"python2.7 -u manage.py update_omim" % locals(),
verbose=True)
run_in_pod(pod_name,
"/usr/local/bin/restart_server.sh" % locals(),
verbose=True)
def print_log(components, deployment_target, enable_stream_log, wait=True):
"""Executes kubernetes command to print logs for the given pod.
Args:
components (list): one or more kubernetes pod labels (eg. 'phenotips' or 'nginx').
If more than one is specified, logs will be printed from all components in parallel.
deployment_target (string): "local", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
enable_stream_log (bool): whether to continuously stream the log instead of just printing
the log up to now.
wait (bool): If False, this method will return without waiting for the log streaming process
to finish printing all logs.
Returns:
(list): Popen process objects for the kubectl port-forward processes.
"""
stream_arg = "-f" if enable_stream_log else ""
procs = []
for component_label in components:
while get_pod_status(component_label, deployment_target) != "Running":
time.sleep(5)
pod_name = get_pod_name(component_label,
deployment_target=deployment_target)
p = run_in_background("kubectl logs %(stream_arg)s %(pod_name)s" %
locals())
def print_command_log():
for line in iter(p.stdout.readline, ''):
logger.info(line.strip('\n'))
t = Thread(target=print_command_log)
t.start()
procs.append(p)
if wait:
wait_for(procs)
return procs
def load_project(deployment_target, project_id="1kg", genome_version="37", vcf=None, ped=None):
"""Load example project
Args:
project_id (string): project id
genome_version (string): reference genome version - either "37" or "38"
vcf (string): VCF path
ped (string): PED path
"""
check_kubernetes_context(deployment_target)
pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not pod_name:
raise ValueError("No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals())
if not project_id:
raise ValueError("project_id not specified")
if not vcf:
raise ValueError("vcf not specified")
if not ped:
raise ValueError("ped not specified")
vcf_filename = os.path.basename(vcf)
ped_filename = os.path.basename(ped)
run_in_pod(pod_name, "wget -N %(vcf)s" % locals())
run_in_pod(pod_name, "wget -N %(ped)s" % locals())
run_in_pod(pod_name, "python2.7 -u manage.py add_project '%(project_id)s' '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_individuals_to_project '%(project_id)s' --ped '%(ped_filename)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_vcf_to_project --clear '%(project_id)s' '%(vcf_filename)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_project_to_phenotips '%(project_id)s' '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_individuals_to_phenotips '%(project_id)s' --ped '%(ped_filename)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py generate_pedigree_images -f '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_default_tags '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py load_project '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py load_project_datastore '%(project_id)s'" % locals(), verbose=True)
def deploy_phenotips(settings):
print_separator("phenotips")
phenotips_service_port = settings["PHENOTIPS_SERVICE_PORT"]
restore_phenotips_db_from_backup = settings.get(
"RESTORE_PHENOTIPS_DB_FROM_BACKUP")
reset_db = settings.get("RESET_DB")
deployment_target = settings["DEPLOY_TO"]
if reset_db or restore_phenotips_db_from_backup:
delete_pod("phenotips", settings)
run_in_pod(
"postgres",
"psql -U postgres postgres -c 'drop database xwiki'" % locals(),
verbose=True,
errors_to_ignore=["does not exist"],
deployment_target=deployment_target,
)
elif settings["DELETE_BEFORE_DEPLOY"]:
delete_pod("phenotips", settings)
run_in_pod(
"postgres",
"psql -U postgres postgres -c \"create role xwiki with CREATEDB LOGIN PASSWORD 'xwiki'\""
% locals(),
verbose=True,
errors_to_ignore=["already exists"],
deployment_target=deployment_target,
)
run_in_pod(
"postgres",
"psql -U xwiki postgres -c 'create database xwiki'" % locals(),
verbose=True,
errors_to_ignore=["already exists"],
deployment_target=deployment_target,
)
run_in_pod(
"postgres",
"psql -U postgres postgres -c 'grant all privileges on database xwiki to xwiki'"
% locals(),
)
docker_build(
"phenotips",
settings,
["--build-arg PHENOTIPS_SERVICE_PORT=%s" % phenotips_service_port],
)
_deploy_pod("phenotips", settings, wait_until_pod_is_ready=True)
for i in range(0, 3):
# opening the PhenoTips website for the 1st time triggers a final set of initialization
# steps which take ~ 1 minute, so run wget to trigger this
try:
run_in_pod(
"phenotips",
#command="wget http://localhost:%(phenotips_service_port)s -O test.html" % locals(),
command=
"curl --verbose -L -u Admin:admin http://localhost:%(phenotips_service_port)s -o test.html"
% locals(),
verbose=True)
except Exception as e:
logger.error(str(e))
if i < 2:
logger.info("Waiting for phenotips to start up...")
time.sleep(10)
if restore_phenotips_db_from_backup:
delete_pod("phenotips", settings)
postgres_pod_name = get_pod_name("postgres",
deployment_target=deployment_target)
run("kubectl cp '%(restore_phenotips_db_from_backup)s' %(postgres_pod_name)s:/root/$(basename %(restore_phenotips_db_from_backup)s)"
% locals(),
verbose=True)
run_in_pod(
"postgres",
"/root/restore_database_backup.sh xwiki xwiki /root/$(basename %(restore_phenotips_db_from_backup)s)"
% locals(),
deployment_target=deployment_target,
verbose=True)
run_in_pod(
"postgres",
"rm /root/$(basename %(restore_phenotips_db_from_backup)s)" %
locals(),
deployment_target=deployment_target,
verbose=True)
_deploy_pod("phenotips", settings, wait_until_pod_is_ready=True)
