def load_example_project(deployment_target, genome_version="37"):
"""Load example project
Args:
genome_version (string): reference genome version - either "37" or "38"
"""
check_kubernetes_context(deployment_target)
pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not pod_name:
raise ValueError("No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals())
if genome_version == "37":
vcf_filename = "1kg.vep.vcf.gz"
elif genome_version == "38":
vcf_filename = "1kg.liftover.GRCh38.vep.vcf.gz"
else:
raise ValueError("Unexpected genome_version: %s" % (genome_version,))
project_id = "1kg"
vcf = "https://storage.googleapis.com/seqr-public/test-projects/1kg-exomes/%(vcf_filename)s" % locals()
ped = "https://storage.googleapis.com/seqr-public/test-projects/1kg-exomes/1kg.ped"
load_project(deployment_target, project_id=project_id, genome_version=genome_version, vcf=vcf, ped=ped)
def load_allele_frequencies(deployment_target, genome_version="37"):
"""Load ExAC and 1kg allele frequency datasets. These are larger and take longer to load than other reference data
Args:
deployment_target (string): "minikube", "gcloud-dev", etc. See constants.DEPLOYMENT_TARGETS.
genome_version (string): reference genome version - either "37" or "38"
"""
check_kubernetes_context(deployment_target)
pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not pod_name:
raise ValueError(
"No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?"
% locals())
run_in_pod(
pod_name,
"wget -N http://seqr.broadinstitute.org/static/bundle/ExAC.r0.3.sites.vep.popmax.clinvar.vcf.gz -P /data/reference_data/"
% locals(),
verbose=True)
run_in_pod(
pod_name,
"wget -N http://seqr.broadinstitute.org/static/bundle/ALL.wgs.phase3_shapeit2_mvncall_integrated_v5a.20130502.sites.decomposed.with_popmax.vcf.gz -P /data/reference_data/"
% locals(),
verbose=True)
run_in_pod(pod_name,
"python2.7 -u manage.py load_reference" % locals(),
verbose=True)
def load_reference_data(deployment_target, genome_version="37"):
"""Load reference data
Args:
genome_version (string): reference genome version - either "37" or "38"
"""
check_kubernetes_context(deployment_target)
pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not pod_name:
raise ValueError(
"No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?"
% locals())
run_in_pod(pod_name, "mkdir -p /data/reference_data/" % locals())
run_in_pod(
pod_name,
"wget -N https://storage.googleapis.com/seqr-public/reference-data/seqr-resource-bundle.GRCh%(genome_version)s.tar.gz -P /data/reference_data/"
% locals(),
verbose=True)
run_in_pod(
pod_name,
"tar -xzf /data/reference_data/seqr-resource-bundle.GRCh%(genome_version)s.tar.gz --directory /data/reference_data/"
% locals(),
verbose=True)
run_in_pod(pod_name,
"python2.7 -u manage.py load_resources" % locals(),
verbose=True)
run_in_pod(
pod_name,
"python2.7 -u manage.py update_gencode /data/reference_data/gencode.v19.annotation.gtf.gz"
% locals(),
verbose=True)
run_in_pod(pod_name,
"python2.7 -u manage.py update_human_phenotype_ontology" %
locals(),
verbose=True)
run_in_pod(pod_name,
"python2.7 -u manage.py update_omim" % locals(),
verbose=True)
run_in_pod(pod_name,
"/usr/local/bin/restart_server.sh" % locals(),
verbose=True)
def load_project(deployment_target, project_id="1kg", genome_version="37", vcf=None, ped=None):
"""Load example project
Args:
project_id (string): project id
genome_version (string): reference genome version - either "37" or "38"
vcf (string): VCF path
ped (string): PED path
"""
check_kubernetes_context(deployment_target)
pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not pod_name:
raise ValueError("No 'seqr' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?" % locals())
if not project_id:
raise ValueError("project_id not specified")
if not vcf:
raise ValueError("vcf not specified")
if not ped:
raise ValueError("ped not specified")
vcf_filename = os.path.basename(vcf)
ped_filename = os.path.basename(ped)
run_in_pod(pod_name, "wget -N %(vcf)s" % locals())
run_in_pod(pod_name, "wget -N %(ped)s" % locals())
run_in_pod(pod_name, "python2.7 -u manage.py add_project '%(project_id)s' '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_individuals_to_project '%(project_id)s' --ped '%(ped_filename)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_vcf_to_project --clear '%(project_id)s' '%(vcf_filename)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_project_to_phenotips '%(project_id)s' '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_individuals_to_phenotips '%(project_id)s' --ped '%(ped_filename)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py generate_pedigree_images -f '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py add_default_tags '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py load_project '%(project_id)s'" % locals(), verbose=True)
run_in_pod(pod_name, "python2.7 -u manage.py load_project_datastore '%(project_id)s'" % locals(), verbose=True)
def deploy(deployment_target, components, output_dir=None, other_settings={}):
"""Deploy all seqr components to a kubernetes cluster.
Args:
deployment_target (string): one of the DEPLOYMENT_TARGETs (eg. "local", or "gcloud")
components (list): A list of components to be deployed from constants.DEPLOYABLE_COMPONENTS
(eg. "postgres", "phenotips").
output_dir (string): path of directory where to put deployment logs and rendered config files
other_settings (dict): a dictionary of other key-value pairs for use during deployment
"""
if not components:
raise ValueError("components list is empty")
if components and "init-cluster" not in components:
check_kubernetes_context(deployment_target)
# parse settings files
settings = retrieve_settings(deployment_target)
settings.update(other_settings)
# configure deployment dir
timestamp = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime())
settings["DEPLOYMENT_TEMP_DIR"] = os.path.join(
settings["DEPLOYMENT_TEMP_DIR"],
"deployments/%(timestamp)s_%(deployment_target)s" % locals())
# configure logging output
log_dir = os.path.join(settings["DEPLOYMENT_TEMP_DIR"], "logs")
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_file_path = os.path.join(log_dir, "deploy.log")
sh = logging.StreamHandler(open(log_file_path, "w"))
sh.setLevel(logging.INFO)
logger.addHandler(sh)
logger.info("Starting log file: %(log_file_path)s" % locals())
# upper-case settings keys
for key, value in settings.items():
key = key.upper()
settings[key] = value
logger.info("%s = %s" % (key, value))
# render Jinja templates and put results in output directory
for file_path in glob.glob("deploy/kubernetes/*.yaml") + glob.glob(
"deploy/kubernetes/*/*.yaml"):
file_path = file_path.replace('deploy/kubernetes/', '')
input_base_dir = os.path.join(other_settings["BASE_DIR"],
'deploy/kubernetes')
output_base_dir = os.path.join(settings["DEPLOYMENT_TEMP_DIR"],
'deploy/kubernetes')
render(input_base_dir, file_path, settings, output_base_dir)
# deploy
if "init-cluster" in components:
deploy_init_cluster(settings)
for retry_i in range(1, 5):
try:
deploy_config_map(settings)
break
except RuntimeError as e:
logger.error((
"Error when deploying config maps: %(e)s. This sometimes happens when cluster is "
"initializing. Retrying...") % locals())
time.sleep(5)
if "secrets" in components:
deploy_secrets(settings)
#if "init-elasticsearch-cluster" in components:
# pass
if "cockpit" in components:
deploy_cockpit(settings)
if "mongo" in components:
deploy_mongo(settings)
if "postgres" in components:
deploy_postgres(settings)
if "phenotips" in components:
deploy_phenotips(settings)
if "matchbox" in components:
deploy_matchbox(settings)
if "seqr" in components:
deploy_seqr(settings)
if "elasticsearch" in components:
deploy_elasticsearch(settings)
if "es-client" in components:
deploy_elasticsearch_sharded("es-client", settings)
if "es-master" in components:
deploy_elasticsearch_sharded("es-master", settings)
if "es-data" in components:
deploy_elasticsearch_sharded("es-data", settings)
if "es-kibana" in components:
deploy_elasticsearch_sharded("kibana", settings)
if "kibana" in components:
deploy_kibana(settings)
if "nginx" in components:
deploy_nginx(settings)
if "pipeline-runner" in components:
deploy_pipeline_runner(settings)
