def _helm_chart_helper(namespace,
should_cleanup,
helm_config,
helm_install_name,
chart_name="helm/dagster"):
"""Install helm chart."""
check.str_param(namespace, "namespace")
check.bool_param(should_cleanup, "should_cleanup")
check.str_param(helm_install_name, "helm_install_name")
print("--- \033[32m:helm: Installing Helm chart {}\033[0m".format(
helm_install_name))
try:
helm_config_yaml = yaml.dump(helm_config, default_flow_style=False)
release_name = chart_name.split("/")[-1]
helm_cmd = [
"helm",
"install",
"--namespace",
namespace,
"--debug",
"-f",
"-",
release_name,
os.path.join(git_repository_root(), chart_name),
]
print("Running Helm Install: \n", " ".join(helm_cmd),
"\nWith config:\n", helm_config_yaml)
p = subprocess.Popen(helm_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate(helm_config_yaml.encode("utf-8"))
print("Helm install completed with stdout: ", stdout.decode("utf-8"))
print("Helm install completed with stderr: ", stderr.decode("utf-8"))
assert p.returncode == 0
# Wait for Dagit pod to be ready (won't actually stay up w/out js rebuild)
kube_api = kubernetes.client.CoreV1Api()
if chart_name == "helm/dagster":
print("Waiting for Dagit pod to be ready...")
dagit_pod = None
while dagit_pod is None:
pods = kube_api.list_namespaced_pod(namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items
if "dagit" in p.metadata.name
]
if pod_names:
dagit_pod = pod_names[0]
time.sleep(1)
# Wait for Celery worker queues to become ready
pods = kubernetes.client.CoreV1Api().list_namespaced_pod(
namespace=namespace)
deployments = kubernetes.client.AppsV1Api(
).list_namespaced_deployment(namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items
if CELERY_WORKER_NAME_PREFIX in p.metadata.name
]
if helm_config.get("runLauncher").get(
"type") == "CeleryK8sRunLauncher":
worker_queues = (helm_config.get("runLauncher").get(
"config").get("celeryK8sRunLauncher").get(
"workerQueues", []))
for queue in worker_queues:
num_pods_for_queue = len([
pod_name for pod_name in pod_names
if f"{CELERY_WORKER_NAME_PREFIX}-{queue.get('name')}"
in pod_name
])
assert queue.get("replicaCount") == num_pods_for_queue
labels = queue.get("labels")
if labels:
target_deployments = []
for item in deployments.items:
if queue.get("name") in item.metadata.name:
target_deployments.append(item)
assert len(target_deployments) > 0
for target in target_deployments:
for key in labels:
assert target.spec.template.metadata.labels.get(
key) == labels.get(key)
print("Waiting for celery workers")
for pod_name in pod_names:
print("Waiting for Celery worker pod %s" % pod_name)
wait_for_pod(pod_name, namespace=namespace)
rabbitmq_enabled = ("rabbitmq" not in helm_config
) or helm_config.get("rabbitmq")
if rabbitmq_enabled:
print("Waiting for rabbitmq pod to exist...")
while True:
pods = kube_api.list_namespaced_pod(
namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items
if "rabbitmq" in p.metadata.name
]
if pod_names:
assert len(pod_names) == 1
print("Waiting for rabbitmq pod to be ready: " +
str(pod_names[0]))
wait_for_pod(pod_names[0], namespace=namespace)
break
time.sleep(1)
else:
assert (
len(pod_names) == 0
), "celery-worker pods {pod_names} exists when celery is not enabled.".format(
pod_names=pod_names)
dagster_user_deployments_values = helm_config.get(
"dagster-user-deployments", {})
if (dagster_user_deployments_values.get("enabled")
and dagster_user_deployments_values.get("enableSubchart")
or release_name == "dagster"):
# Wait for user code deployments to be ready
print("Waiting for user code deployments")
pods = kubernetes.client.CoreV1Api().list_namespaced_pod(
namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items
if "user-code-deployment" in p.metadata.name
]
for pod_name in pod_names:
print("Waiting for user code deployment pod %s" % pod_name)
wait_for_pod(pod_name, namespace=namespace)
print("Helm chart successfully installed in namespace %s" % namespace)
yield
finally:
# Can skip this step as a time saver when we're going to destroy the cluster anyway, e.g.
# w/ a kind cluster
if should_cleanup:
print("Uninstalling helm chart")
check_output(
["helm", "uninstall", release_name, "--namespace", namespace],
cwd=git_repository_root(),
)
def _helm_chart_helper(namespace, should_cleanup, helm_config):
"""Install helm chart.
"""
check.str_param(namespace, "namespace")
check.bool_param(should_cleanup, "should_cleanup")
print("--- \033[32m:helm: Installing Helm chart\033[0m")
try:
helm_config_yaml = yaml.dump(helm_config, default_flow_style=False)
helm_cmd = [
"helm",
"install",
"--namespace",
namespace,
"-f",
"-",
"dagster",
os.path.join(git_repository_root(), "helm", "dagster"),
]
print("Running Helm Install: \n", " ".join(helm_cmd),
"\nWith config:\n", helm_config_yaml)
p = subprocess.Popen(helm_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate(six.ensure_binary(helm_config_yaml))
print("Helm install completed with stdout: ", stdout)
print("Helm install completed with stderr: ", stderr)
assert p.returncode == 0
# Wait for Dagit pod to be ready (won't actually stay up w/out js rebuild)
kube_api = kubernetes.client.CoreV1Api()
print("Waiting for Dagit pod to be ready...")
dagit_pod = None
while dagit_pod is None:
pods = kube_api.list_namespaced_pod(namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items
if "dagit" in p.metadata.name
]
if pod_names:
dagit_pod = pod_names[0]
time.sleep(1)
# Wait for Celery worker queues to become ready
print("Waiting for celery workers")
pods = kubernetes.client.CoreV1Api().list_namespaced_pod(
namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items
if "celery-workers" in p.metadata.name
]
for pod_name in pod_names:
print("Waiting for Celery worker pod %s" % pod_name)
wait_for_pod(pod_name, namespace=namespace)
if helm_config.get("userDeployments") and helm_config.get(
"userDeployments", {}).get("enabled"):
# Wait for user code deployments to be ready
print("Waiting for user code deployments")
pods = kubernetes.client.CoreV1Api().list_namespaced_pod(
namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items
if "user-code-deployment" in p.metadata.name
]
for pod_name in pod_names:
print("Waiting for user code deployment pod %s" % pod_name)
wait_for_pod(pod_name, namespace=namespace)
yield
finally:
# Can skip this step as a time saver when we're going to destroy the cluster anyway, e.g.
# w/ a kind cluster
if should_cleanup:
print("Uninstalling helm chart")
check_output(
["helm", "uninstall", "dagster", "--namespace", namespace],
cwd=git_repository_root(),
)
import os
import sys
import pytest
from dagster_k8s_tests.utils import wait_for_job_and_get_logs
from dagster_test.test_project import (
get_test_project_external_pipeline,
test_project_environments_path,
)
from dagster.core.test_utils import create_run_for_test
from dagster.utils import git_repository_root, merge_dicts
from dagster.utils.yaml_utils import merge_yamls
sys.path.append(
os.path.join(git_repository_root(), 'python_modules', 'libraries',
'dagster-k8s'))
from dagster_k8s_tests.cluster import ( # isort:skip, pylint:disable=unused-import
dagster_instance, run_launcher,
)
@pytest.mark.integration
@pytest.mark.skipif(sys.version_info < (3, 5), reason="Very slow on Python 2")
def test_execute_on_celery( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace):
environment_dict = merge_dicts(
merge_yamls([
os.path.join(test_project_environments_path(), 'env.yaml'),
os.path.join(test_project_environments_path(), 'env_s3.yaml'),
]),
def test_repo_path():
return os.path.join(
git_repository_root(), 'python_modules', 'dagster-test', 'dagster_test', 'test_project'
)
def get_test_repo_path():
return os.path.join(git_repository_root(), "python_modules",
"dagster-test", "dagster_test", "test_project")
def test_repo_path():
return os.path.join(git_repository_root(), '.buildkite', 'images',
'docker', 'test_project')
from airflow.exceptions import AirflowException
from airflow.utils import timezone
from dagster_airflow.factory import make_airflow_dag_kubernetized_for_handle
from dagster_airflow.test_fixtures import ( # pylint: disable=unused-import
dagster_airflow_k8s_operator_pipeline,
execute_tasks_in_dag,
)
from dagster_airflow_tests.marks import nettest
from dagster.core.definitions.reconstructable import ReconstructableRepository
from dagster.core.utils import make_new_run_id
from dagster.utils import git_repository_root, load_yaml_from_glob_list
from .utils import validate_pipeline_execution
sys.path.append(os.path.join(git_repository_root(), 'python_modules', 'libraries', 'dagster-k8s'))
from dagster_k8s_tests.test_project import test_project_environments_path # isort:skip
@nettest
def test_s3_storage(dagster_airflow_k8s_operator_pipeline, dagster_docker_image, cluster_provider):
print('--- :airflow: test_kubernetes.test_s3_storage')
_check_aws_creds_available()
environments_path = test_project_environments_path()
pipeline_name = 'demo_pipeline'
results = dagster_airflow_k8s_operator_pipeline(
pipeline_name=pipeline_name,
handle=ReconstructableRepository.for_module('test_pipelines.repo', pipeline_name),
environment_yaml=[
os.path.join(environments_path, 'env.yaml'),
def _helm_chart_helper(namespace, should_cleanup, helm_config, helm_install_name):
"""Install helm chart.
"""
check.str_param(namespace, "namespace")
check.bool_param(should_cleanup, "should_cleanup")
check.str_param(helm_install_name, "helm_install_name")
print("--- \033[32m:helm: Installing Helm chart {}\033[0m".format(helm_install_name))
try:
helm_config_yaml = yaml.dump(helm_config, default_flow_style=False)
helm_cmd = [
"helm",
"install",
"--namespace",
namespace,
"-f",
"-",
"dagster",
os.path.join(git_repository_root(), "helm", "dagster"),
]
print("Running Helm Install: \n", " ".join(helm_cmd), "\nWith config:\n", helm_config_yaml)
p = subprocess.Popen(
helm_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout, stderr = p.communicate(six.ensure_binary(helm_config_yaml))
print("Helm install completed with stdout: ", stdout)
print("Helm install completed with stderr: ", stderr)
assert p.returncode == 0
# Wait for Dagit pod to be ready (won't actually stay up w/out js rebuild)
kube_api = kubernetes.client.CoreV1Api()
print("Waiting for Dagit pod to be ready...")
dagit_pod = None
while dagit_pod is None:
pods = kube_api.list_namespaced_pod(namespace=namespace)
pod_names = [p.metadata.name for p in pods.items if "dagit" in p.metadata.name]
if pod_names:
dagit_pod = pod_names[0]
time.sleep(1)
# Wait for Celery worker queues to become ready
pods = kubernetes.client.CoreV1Api().list_namespaced_pod(namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items if CELERY_WORKER_NAME_PREFIX in p.metadata.name
]
if helm_config.get("celery", {}).get("enabled"):
worker_queues = helm_config.get("celery").get("workerQueues", [])
for queue in worker_queues:
num_pods_for_queue = len(
[
pod_name
for pod_name in pod_names
if f"{CELERY_WORKER_NAME_PREFIX}-{queue.get('name')}" in pod_name
]
)
assert queue.get("replicaCount") == num_pods_for_queue
print("Waiting for celery workers")
for pod_name in pod_names:
print("Waiting for Celery worker pod %s" % pod_name)
wait_for_pod(pod_name, namespace=namespace)
rabbitmq_enabled = ("rabbitmq" not in helm_config) or helm_config.get("rabbitmq")
if rabbitmq_enabled:
print("Waiting for rabbitmq pod to exist...")
while True:
pods = kube_api.list_namespaced_pod(namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items if "rabbitmq" in p.metadata.name
]
if pod_names:
assert len(pod_names) == 1
print("Waiting for rabbitmq pod to be ready: " + str(pod_names[0]))
wait_for_pod(pod_names[0], namespace=namespace)
break
time.sleep(1)
else:
assert (
len(pod_names) == 0
), "celery-worker pods {pod_names} exists when celery is not enabled.".format(
pod_names=pod_names
)
if helm_config.get("userDeployments") and helm_config.get("userDeployments", {}).get(
"enabled"
):
# Wait for user code deployments to be ready
print("Waiting for user code deployments")
pods = kubernetes.client.CoreV1Api().list_namespaced_pod(namespace=namespace)
pod_names = [
p.metadata.name for p in pods.items if "user-code-deployment" in p.metadata.name
]
for pod_name in pod_names:
print("Waiting for user code deployment pod %s" % pod_name)
wait_for_pod(pod_name, namespace=namespace)
yield
finally:
# Can skip this step as a time saver when we're going to destroy the cluster anyway, e.g.
# w/ a kind cluster
if should_cleanup:
print("Uninstalling helm chart")
check_output(
["helm", "uninstall", "dagster", "--namespace", namespace],
cwd=git_repository_root(),
)
