Esempio n. 1
0
def deploy_seqr(settings):
    print_separator("seqr")

    if settings["BUILD_DOCKER_IMAGES"]:
        seqr_git_hash = run("git log -1 --pretty=%h",
                            errors_to_ignore=["Not a git repository"])
        seqr_git_hash = (
            ":" + seqr_git_hash.strip()) if seqr_git_hash is not None else ""

        docker_build("seqr", settings, [
            "--build-arg SEQR_SERVICE_PORT=%s" % settings["SEQR_SERVICE_PORT"],
            "--build-arg SEQR_UI_DEV_PORT=%s" % settings["SEQR_UI_DEV_PORT"],
            "-f deploy/docker/seqr/Dockerfile",
            "-t %(DOCKER_IMAGE_NAME)s" + seqr_git_hash,
        ])

    if settings["ONLY_PUSH_TO_REGISTRY"]:
        return

    restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP")
    reset_db = settings.get("RESET_DB")

    deployment_target = settings["DEPLOY_TO"]
    postgres_pod_name = get_pod_name("postgres",
                                     deployment_target=deployment_target)

    if settings["DELETE_BEFORE_DEPLOY"]:
        delete_pod("seqr", settings)
    elif reset_db or restore_seqr_db_from_backup:
        seqr_pod_name = get_pod_name('seqr',
                                     deployment_target=deployment_target)
        if seqr_pod_name:
            sleep_until_pod_is_running("seqr",
                                       deployment_target=deployment_target)

            run_in_pod(seqr_pod_name,
                       "/usr/local/bin/stop_server.sh",
                       verbose=True)

    if reset_db:
        _drop_seqr_db(postgres_pod_name)
    if restore_seqr_db_from_backup:
        _drop_seqr_db(postgres_pod_name)
        _restore_seqr_db_from_backup(postgres_pod_name,
                                     restore_seqr_db_from_backup)
    else:
        run_in_pod(
            postgres_pod_name,
            "psql -U postgres postgres -c 'create database seqrdb'",
            errors_to_ignore=["already exists"],
            verbose=True,
        )
        run_in_pod(
            postgres_pod_name,
            "psql -U postgres postgres -c 'create database reference_data_db'",
            errors_to_ignore=["already exists"],
            verbose=True,
        )

    deploy_pod("seqr", settings, wait_until_pod_is_ready=True)
Esempio n. 2
0
def deploy_postgres(settings):
    print_separator("postgres")

    docker_build("postgres", settings)

    restore_seqr_db_from_backup = settings.get("RESTORE_SEQR_DB_FROM_BACKUP")
    reset_db = settings.get("RESET_DB")

    if reset_db or restore_seqr_db_from_backup:
        # Since pgdata is stored on a persistent volume, redeploying does not get rid of it. If any existing pgdata is
        # present, even if the databases are empty, postgres will not fully re-initialize the database. This is
        # good if you want to keep the data across a deployment, but problematic if you actually need to rest and
        # reinitialize the db. Therefore, when the database needs to be fully reinitialized, delete pgdata
        run_in_pod(get_pod_name("postgres",
                                deployment_target=settings["DEPLOY_TO"]),
                   "rm -rf /var/lib/postgresql/data/pgdata",
                   verbose=True)

    deploy_pod("postgres", settings, wait_until_pod_is_ready=True)

    if restore_seqr_db_from_backup:
        postgres_pod_name = get_pod_name(
            "postgres", deployment_target=settings["DEPLOY_TO"])
        _restore_seqr_db_from_backup(
            postgres_pod_name, restore_seqr_db_from_backup,
            settings.get("RESTORE_REFERENCE_DB_FROM_BACKUP"))
Esempio n. 3
0
def copy_files_to_or_from_pod(component,
                              deployment_target,
                              source_path,
                              dest_path,
                              direction=1):
    """Copy file(s) to or from the given component.

    Args:
        component (string): component label (eg. "postgres")
        deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
        source_path (string): source file path. If copying files to the component, it should be a local path. Otherwise, it should be a file path inside the component pod.
        dest_path (string): destination file path. If copying files from the component, it should be a local path. Otherwise, it should be a file path inside the component pod.
        direction (int): If > 0 the file will be copied to the pod. If < 0, then it will be copied from the pod.
    """
    full_pod_name = get_pod_name(component,
                                 deployment_target=deployment_target)
    if not full_pod_name:
        raise ValueError(
            "No '%(pod_name)s' pods found. Is the kubectl environment configured in this terminal? and has this type of pod been deployed?"
            % locals())

    if direction < 0:  # copy from pod
        source_path = "%s:%s" % (full_pod_name, source_path)
    elif direction > 0:  # copy to pod
        dest_path = "%s:%s" % (full_pod_name, dest_path)

    run("kubectl cp '%(source_path)s' '%(dest_path)s'" % locals())
Esempio n. 4
0
def delete_component(component, deployment_target=None):
    """Runs kubectl commands to delete any running deployment, service, or pod objects for the given component(s).

    Args:
        component (string): component to delete (eg. 'postgres' or 'nginx').
        deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
    """
    if component == "cockpit":
        run("kubectl delete rc cockpit", errors_to_ignore=["not found"])
    elif component == "es-data":
        run("kubectl delete StatefulSet es-data", errors_to_ignore=["not found"])
    elif component == "nginx":
        raise ValueError("TODO: implement deleting nginx")

    run("kubectl delete deployments %(component)s" % locals(), errors_to_ignore=["not found"])
    run("kubectl delete services %(component)s" % locals(), errors_to_ignore=["not found"])

    pod_name = get_pod_name(component, deployment_target=deployment_target)
    if pod_name:
        run("kubectl delete pods %(pod_name)s" % locals(), errors_to_ignore=["not found"])

        logger.info("waiting for \"%s\" to exit Running status" % component)
        while is_pod_running(component, deployment_target):
            time.sleep(5)


    # print services and pods status
    run("kubectl get services" % locals(), verbose=True)
    run("kubectl get pods" % locals(), verbose=True)
Esempio n. 5
0
def troubleshoot_component(component, deployment_target):
    """Runs kubectl command to print detailed debug output for the given component.

    Args:
        component (string): component label (eg. "postgres")
        deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
    """

    pod_name = get_pod_name(component, deployment_target=deployment_target)

    run("kubectl get pods -o yaml %(pod_name)s" % locals(), verbose=True)
Esempio n. 6
0
def redeploy_seqr(deployment_target):
    print_separator('re-deploying seqr')

    seqr_pod_name = get_pod_name('seqr', deployment_target=deployment_target)
    if not seqr_pod_name:
        raise ValueError('No seqr pod found, unable to re-deploy')
    sleep_until_pod_is_running('seqr', deployment_target=deployment_target)

    run_in_pod(seqr_pod_name, 'git pull', verbose=True)
    run_in_pod(seqr_pod_name, './manage.py migrate', verbose=True)
    run_in_pod(seqr_pod_name, '/usr/local/bin/restart_server.sh')
Esempio n. 7
0
def print_log(components,
              deployment_target,
              enable_stream_log,
              previous=False,
              wait=True):
    """Executes kubernetes command to print logs for the given pod.

    Args:
        components (list): one or more kubernetes pod labels (eg. 'postgres' or 'nginx').
            If more than one is specified, logs will be printed from all components in parallel.
        deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev", etc.
        enable_stream_log (bool): whether to continuously stream the log instead of just printing
            the log up to now.
        previous (bool): Prints logs from a previous instance of the container. This is useful for debugging pods that
            don't start or immediately enter crash-loop.
        wait (bool): If False, this method will return without waiting for the log streaming process
            to finish printing all logs.

    Returns:
        (list): Popen process objects for the kubectl port-forward processes.
    """
    stream_arg = "-f" if enable_stream_log else ""
    previous_flag = "--previous" if previous else ""

    procs = []
    for component_label in components:
        if component_label == "kube-scan":
            continue  # See https://github.com/octarinesec/kube-scan for how to connect to the kube-scan pod.

        if not previous:
            wait_until_pod_is_running(component_label, deployment_target)

        pod_name = get_pod_name(component_label,
                                deployment_target=deployment_target)

        p = run_in_background(
            "kubectl logs %(stream_arg)s %(previous_flag)s %(pod_name)s --all-containers"
            % locals(),
            print_command=True)

        def print_command_log():
            for line in iter(p.stdout.readline, ''):
                logger.info(line.strip('\n'))

        t = Thread(target=print_command_log)
        t.start()
        procs.append(p)

    if wait:
        wait_for(procs)

    return procs
Esempio n. 8
0
def reset_database(database=[], deployment_target=None):
    """Runs kubectl commands to delete and reset the given database(s).

    Args:
        component (list): one more database labels - "seqrdb"
        deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
    """
    if "seqrdb" in database:
        postgres_pod_name = get_pod_name("postgres", deployment_target=deployment_target)
        if not postgres_pod_name:
            logger.error("postgres pod must be running")
        else:
            run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'drop database seqrdb'" % locals(), errors_to_ignore=["does not exist"])
            run_in_pod(postgres_pod_name, "psql -U postgres postgres -c 'create database seqrdb'" % locals())
Esempio n. 9
0
def port_forward(component_port_pairs=[], deployment_target=None, wait=True, open_browser=False, use_kubectl_proxy=False):
    """Executes kubectl command to set up port forwarding between localhost and the given pod.
    While this is running, connecting to localhost:<port> will be the same as connecting to that port
    from the pod's internal network.

    Args:
        component_port_pairs (list): 2-tuple(s) containing keyword to use for looking up a kubernetes
            pod, along with the port to forward to that pod (eg. ('postgres', 5432))
        deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
        wait (bool): Whether to block indefinitely as long as the forwarding process is running.
        open_browser (bool): If component_port_pairs includes components that have an http server
            (eg. "seqr" or "postgres"), then open a web browser window to the forwarded port.
        use_kubectl_proxy (bool): Whether to use kubectl proxy instead of kubectl port-forward
            (see https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#manually-constructing-apiserver-proxy-urls)
    Returns:
        (list): Popen process objects for the kubectl port-forward processes.
    """
    procs = []
    for component_label, port in component_port_pairs:
        if component_label == "kube-scan":
            continue  # See https://github.com/octarinesec/kube-scan for how to connect to the kube-scan pod.

        wait_until_pod_is_running(component_label, deployment_target)

        logger.info("Forwarding port %s for %s" % (port, component_label))
        pod_name = get_pod_name(component_label, deployment_target=deployment_target)

        if use_kubectl_proxy:
            command = "kubectl proxy --port 8001"
        else:
            command = "kubectl port-forward %(pod_name)s %(port)s" % locals()

        p = run_in_background(command)

        if open_browser and component_label in COMPONENTS_TO_OPEN_IN_BROWSER:
            if use_kubectl_proxy:
                url = "http://localhost:8001/api/v1/namespaces/default/services/%(component_label)s:%(port)s/proxy/" % locals()
            else:
                url = "http://localhost:%s" % port

            time.sleep(3)
            os.system("open " + url)

        procs.append(p)

    if wait:
        wait_for(procs)

    return procs
Esempio n. 10
0
def delete_component(component, deployment_target=None):
    """Runs kubectl commands to delete any running deployment, service, or pod objects for the given component(s).

    Args:
        component (string): component to delete (eg. 'postgres' or 'nginx').
        deployment_target (string): value from DEPLOYMENT_TARGETS - eg. "gcloud-dev"
    """
    if component == "cockpit":
        run("kubectl delete rc cockpit", errors_to_ignore=["not found"])
    elif component == 'elasticsearch':
        run('kubectl delete elasticsearch elasticsearch',
            errors_to_ignore=['not found'])
        # Deleting a released persistent volume does not delete the data on the underlying disk
        wait_for_resource(component,
                          '{.items[0].status.phase}',
                          'Released',
                          deployment_target=deployment_target,
                          resource_type='pv')
        pv = get_resource_name(component,
                               resource_type='pv',
                               deployment_target=deployment_target)
        while pv:
            run('kubectl delete pv {}'.format(pv))
            pv = get_resource_name(component,
                                   resource_type='pv',
                                   deployment_target=deployment_target)
    elif component == 'kibana':
        run('kubectl delete kibana kibana', errors_to_ignore=['not found'])
    elif component == "nginx":
        raise ValueError("TODO: implement deleting nginx")

    run("kubectl delete deployments %(component)s" % locals(),
        errors_to_ignore=["not found"])
    run("kubectl delete services %(component)s" % locals(),
        errors_to_ignore=["not found"])

    pod_name = get_pod_name(component, deployment_target=deployment_target)
    if pod_name:
        run("kubectl delete pods %(pod_name)s" % locals(),
            errors_to_ignore=["not found"])

        logger.info("waiting for \"%s\" to exit Running status" % component)
        while is_pod_running(component, deployment_target):
            time.sleep(5)

    # print services and pods status
    run("kubectl get services" % locals(), verbose=True)
    run("kubectl get pods" % locals(), verbose=True)