def deploy_seqr(settings):
print_separator("seqr")
if settings["BUILD_DOCKER_IMAGES"]:
seqr_git_hash = run("git log -1 --pretty=%h",
errors_to_ignore=["Not a git repository"])
seqr_git_hash = (
":" + seqr_git_hash.strip()) if seqr_git_hash is not None else ""
docker_build("seqr", settings, [
"--build-arg SEQR_SERVICE_PORT=%s" % settings["SEQR_SERVICE_PORT"],
"--build-arg SEQR_UI_DEV_PORT=%s" % settings["SEQR_UI_DEV_PORT"],
"-f deploy/docker/seqr/Dockerfile",
"-t %(DOCKER_IMAGE_NAME)s" + seqr_git_hash,
])
if settings["ONLY_PUSH_TO_REGISTRY"]:
return
restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP")
reset_db = settings.get("RESET_DB")
deployment_target = settings["DEPLOY_TO"]
postgres_pod_name = get_pod_name("postgres",
deployment_target=deployment_target)
if settings["DELETE_BEFORE_DEPLOY"]:
delete_pod("seqr", settings)
elif reset_db or restore_seqr_db_from_backup:
seqr_pod_name = get_pod_name('seqr',
deployment_target=deployment_target)
if seqr_pod_name:
sleep_until_pod_is_running("seqr",
deployment_target=deployment_target)
run_in_pod(seqr_pod_name,
"/usr/local/bin/stop_server.sh",
verbose=True)
if reset_db:
_drop_seqr_db(postgres_pod_name)
if restore_seqr_db_from_backup:
_drop_seqr_db(postgres_pod_name)
_restore_seqr_db_from_backup(postgres_pod_name,
restore_seqr_db_from_backup)
else:
run_in_pod(
postgres_pod_name,
"psql -U postgres postgres -c 'create database seqrdb'",
errors_to_ignore=["already exists"],
verbose=True,
)
run_in_pod(
postgres_pod_name,
"psql -U postgres postgres -c 'create database reference_data_db'",
errors_to_ignore=["already exists"],
verbose=True,
)
deploy_pod("seqr", settings, wait_until_pod_is_ready=True)
def deploy_postgres(settings):
print_separator("postgres")
docker_build("postgres", settings)
restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP")
reset_db = settings.get("RESET_DB")
if reset_db or restore_seqr_db_from_backup:
# Since pgdata is stored on a persistent volume, redeploying does not get rid of it. If any existing pgdata is
# present, even if the databases are empty, postgres will not fully re-initialize the database. This is
# good if you want to keep the data across a deployment, but problematic if you actually need to rest and
# reinitialize the db. Therefore, when the database needs to be fully reinitialized, delete pgdata
run_in_pod(get_pod_name("postgres",
deployment_target=settings["DEPLOY_TO"]),
"rm -rf /var/lib/postgresql/data/pgdata",
verbose=True)
deploy_pod("postgres", settings, wait_until_pod_is_ready=True)
if restore_seqr_db_from_backup:
postgres_pod_name = get_pod_name(
"postgres", deployment_target=settings["DEPLOY_TO"])
_restore_seqr_db_from_backup(
postgres_pod_name, restore_seqr_db_from_backup,
settings.get("RESTORE_REFERENCE_DB_FROM_BACKUP"))
def copy_files_to_or_from_pod(component,
deployment_target,
source_path,
dest_path,
direction=1):
"""Copy file(s) to or from the given component.
Args:
component (string): component label (eg. "postgres")
deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
source_path (string): source file path. If copying files to the component, it should be a local path. Otherwise, it should be a file path inside the component pod.
dest_path (string): destination file path. If copying files from the component, it should be a local path. Otherwise, it should be a file path inside the component pod.
direction (int): If > 0 the file will be copied to the pod. If < 0, then it will be copied from the pod.
"""
full_pod_name = get_pod_name(component,
deployment_target=deployment_target)
if not full_pod_name:
raise ValueError(
"No '%(pod_name)s' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?"
% locals())
if direction < 0: # copy from pod
source_path = "%s:%s" % (full_pod_name, source_path)
elif direction > 0: # copy to pod
dest_path = "%s:%s" % (full_pod_name, dest_path)
run("kubectl cp '%(source_path)s' '%(dest_path)s'" % locals())
def delete_component(component, deployment_target=None):
"""Runs kubectl commands to delete any running deployment, service, or pod objects for the given component(s).
Args:
component (string): component to delete (eg. 'postgres' or 'nginx').
deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
"""
if component == "cockpit":
run("kubectl delete rc cockpit", errors_to_ignore=["not found"])
elif component == "es-data":
run("kubectl delete StatefulSet es-data", errors_to_ignore=["not found"])
elif component == "nginx":
raise ValueError("TODO: implement deleting nginx")
run("kubectl delete deployments %(component)s" % locals(), errors_to_ignore=["not found"])
run("kubectl delete services %(component)s" % locals(), errors_to_ignore=["not found"])
pod_name = get_pod_name(component, deployment_target=deployment_target)
if pod_name:
run("kubectl delete pods %(pod_name)s" % locals(), errors_to_ignore=["not found"])
logger.info("waiting for \"%s\" to exit Running status" % component)
while is_pod_running(component, deployment_target):
time.sleep(5)
# print services and pods status
run("kubectl get services" % locals(), verbose=True)
run("kubectl get pods" % locals(), verbose=True)
def troubleshoot_component(component, deployment_target):
"""Runs kubectl command to print detailed debug output for the given component.
Args:
component (string): component label (eg. "postgres")
deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
"""
pod_name = get_pod_name(component, deployment_target=deployment_target)
run("kubectl get pods -o yaml %(pod_name)s" % locals(), verbose=True)
def redeploy_seqr(deployment_target):
print_separator('re-deploying seqr')
seqr_pod_name = get_pod_name('seqr', deployment_target=deployment_target)
if not seqr_pod_name:
raise ValueError('No seqr pod found, unable to re-deploy')
sleep_until_pod_is_running('seqr', deployment_target=deployment_target)
run_in_pod(seqr_pod_name, 'git pull', verbose=True)
run_in_pod(seqr_pod_name, './manage.py migrate', verbose=True)
run_in_pod(seqr_pod_name, '/usr/local/bin/restart_server.sh')
def print_log(components,
deployment_target,
enable_stream_log,
previous=False,
wait=True):
"""Executes kubernetes command to print logs for the given pod.
Args:
components (list): one or more kubernetes pod labels (eg. 'postgres' or 'nginx').
If more than one is specified, logs will be printed from all components in parallel.
deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev", etc.
enable_stream_log (bool): whether to continuously stream the log instead of just printing
the log up to now.
previous (bool): Prints logs from a previous instance of the container. This is useful for debugging pods that
don't start or immediately enter crash-loop.
wait (bool): If False, this method will return without waiting for the log streaming process
to finish printing all logs.
Returns:
(list): Popen process objects for the kubectl port-forward processes.
"""
stream_arg = "-f" if enable_stream_log else ""
previous_flag = "--previous" if previous else ""
procs = []
for component_label in components:
if component_label == "kube-scan":
continue # See https://github.com/octarinesec/kube-scan for how to connect to the kube-scan pod.
if not previous:
wait_until_pod_is_running(component_label, deployment_target)
pod_name = get_pod_name(component_label,
deployment_target=deployment_target)
p = run_in_background(
"kubectl logs %(stream_arg)s %(previous_flag)s %(pod_name)s --all-containers"
% locals(),
print_command=True)
def print_command_log():
for line in iter(p.stdout.readline, ''):
logger.info(line.strip('\n'))
t = Thread(target=print_command_log)
t.start()
procs.append(p)
if wait:
wait_for(procs)
return procs
def reset_database(database=[], deployment_target=None):
"""Runs kubectl commands to delete and reset the given database(s).
Args:
component (list): one more database labels - "seqrdb"
deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
"""
if "seqrdb" in database:
postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target)
if not postgres_pod_name:
logger.error("postgres pod must be running")
else:
run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(), errors_to_ignore=["does not exist"])
run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals())
def port_forward(component_port_pairs=[], deployment_target=None, wait=True, open_browser=False, use_kubectl_proxy=False):
"""Executes kubectl command to set up port forwarding between localhost and the given pod.
While this is running, connecting to localhost:<port> will be the same as connecting to that port
from the pod's internal network.
Args:
component_port_pairs (list): 2-tuple(s) containing keyword to use for looking up a kubernetes
pod, along with the port to forward to that pod (eg. ('postgres', 5432))
deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
wait (bool): Whether to block indefinitely as long as the forwarding process is running.
open_browser (bool): If component_port_pairs includes components that have an http server
(eg. "seqr" or "postgres"), then open a web browser window to the forwarded port.
use_kubectl_proxy (bool): Whether to use kubectl proxy instead of kubectl port-forward
(see https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#manually-constructing-apiserver-proxy-urls)
Returns:
(list): Popen process objects for the kubectl port-forward processes.
"""
procs = []
for component_label, port in component_port_pairs:
if component_label == "kube-scan":
continue # See https://github.com/octarinesec/kube-scan for how to connect to the kube-scan pod.
wait_until_pod_is_running(component_label, deployment_target)
logger.info("Forwarding port %s for %s" % (port, component_label))
pod_name = get_pod_name(component_label, deployment_target=deployment_target)
if use_kubectl_proxy:
command = "kubectl proxy --port 8001"
else:
command = "kubectl port-forward %(pod_name)s %(port)s" % locals()
p = run_in_background(command)
if open_browser and component_label in COMPONENTS_TO_OPEN_IN_BROWSER:
if use_kubectl_proxy:
url = "http://localhost:8001/api/v1/namespaces/default/services/%(component_label)s:%(port)s/proxy/" % locals()
else:
url = "http://localhost:%s" % port
time.sleep(3)
os.system("open " + url)
procs.append(p)
if wait:
wait_for(procs)
return procs
def delete_component(component, deployment_target=None):
"""Runs kubectl commands to delete any running deployment, service, or pod objects for the given component(s).
Args:
component (string): component to delete (eg. 'postgres' or 'nginx').
deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
"""
if component == "cockpit":
run("kubectl delete rc cockpit", errors_to_ignore=["not found"])
elif component == 'elasticsearch':
run('kubectl delete elasticsearch elasticsearch',
errors_to_ignore=['not found'])
# Deleting a released persistent volume does not delete the data on the underlying disk
wait_for_resource(component,
'{.items[0].status.phase}',
'Released',
deployment_target=deployment_target,
resource_type='pv')
pv = get_resource_name(component,
resource_type='pv',
deployment_target=deployment_target)
while pv:
run('kubectl delete pv {}'.format(pv))
pv = get_resource_name(component,
resource_type='pv',
deployment_target=deployment_target)
elif component == 'kibana':
run('kubectl delete kibana kibana', errors_to_ignore=['not found'])
elif component == "nginx":
raise ValueError("TODO: implement deleting nginx")
run("kubectl delete deployments %(component)s" % locals(),
errors_to_ignore=["not found"])
run("kubectl delete services %(component)s" % locals(),
errors_to_ignore=["not found"])
pod_name = get_pod_name(component, deployment_target=deployment_target)
if pod_name:
run("kubectl delete pods %(pod_name)s" % locals(),
errors_to_ignore=["not found"])
logger.info("waiting for \"%s\" to exit Running status" % component)
while is_pod_running(component, deployment_target):
time.sleep(5)
# print services and pods status
run("kubectl get services" % locals(), verbose=True)
run("kubectl get pods" % locals(), verbose=True)