def test_execute_queued_run_on_celery_k8s( # pylint: disable=redefined-outer-name
dagster_docker_image,
dagster_instance_for_daemon,
helm_namespace_for_daemon,
dagit_url_for_daemon,
):
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_engine_config(
dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace_for_daemon,
),
)
run_id = launch_run_over_graphql(dagit_url_for_daemon,
run_config=run_config,
pipeline_name="demo_pipeline_celery")
wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_daemon)
logs = dagster_instance_for_daemon.all_logs(run_id)
assert_events_in_order(
logs,
[
"PIPELINE_ENQUEUED", "PIPELINE_DEQUEUED", "PIPELINE_STARTING",
"PIPELINE_SUCCESS"
],
)
def test_execute_on_celery_k8s_default( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace):
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
pipeline_name = "demo_pipeline_celery"
run = create_run_for_test(
dagster_instance,
pipeline_name=pipeline_name,
run_config=run_config,
mode="default",
)
with get_test_project_external_pipeline(
pipeline_name) as external_pipeline:
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(external_pipeline),
)
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" %
run.run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(
result)
updated_run = dagster_instance.get_run_by_id(run.run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == dagster_docker_image
def test_k8s_run_launcher(dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher):
run_config = load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
with get_test_project_workspace_and_external_pipeline(
dagster_instance_for_k8s_run_launcher, pipeline_name) as (
workspace,
external_pipeline,
):
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(
external_pipeline)
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
mode="default",
external_pipeline_origin=reoriginated_pipeline.get_external_origin(
),
pipeline_code_origin=reoriginated_pipeline.get_python_origin(),
)
dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(
result)
def test_failing_k8s_run_launcher(dagster_instance, helm_namespace):
run_config = {"blah blah this is wrong": {}}
pipeline_name = "demo_pipeline"
run = create_run_for_test(dagster_instance,
pipeline_name=pipeline_name,
run_config=run_config)
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(
get_test_project_external_pipeline(pipeline_name)),
)
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" %
run.run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format(
result)
event_records = dagster_instance.all_logs(run.run_id)
assert any([
'Undefined field "blah blah this is wrong"' in str(event)
for event in event_records
])
assert any([
'Missing required field "solids"' in str(event)
for event in event_records
])
def _launch_executor_run(
dagit_url,
run_config,
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
pipeline_name="demo_k8s_executor_pipeline",
num_steps=2,
mode="default",
):
run_id = launch_run_over_graphql(dagit_url,
run_config=run_config,
pipeline_name=pipeline_name,
mode=mode)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image(
)
events = dagster_instance_for_k8s_run_launcher.all_logs(run_id)
assert len(_get_step_execution_events(events)) == num_steps
return run_id
def test_k8s_run_launcher_default(
dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagster_docker_image
):
# sanity check that we have a K8sRunLauncher
check.inst(dagster_instance_for_k8s_run_launcher.run_launcher, K8sRunLauncher)
pods = DagsterKubernetesClient.production_client().core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher
)
celery_pod_names = [p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name]
check.invariant(not celery_pod_names)
run_config = merge_dicts(
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env.yaml")),
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
"env_config_maps": ["dagster-pipeline-env"]
+ ([TEST_AWS_CONFIGMAP_NAME] if not IS_BUILDKITE else []),
}
}
},
},
)
pipeline_name = "demo_k8s_executor_pipeline"
tags = {"key": "value"}
with get_test_project_location_and_external_pipeline(pipeline_name) as (
location,
external_pipeline,
):
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
tags=tags,
mode="default",
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=location.get_external_execution_plan(
external_pipeline, run_config, "default", None, None
).execution_plan_snapshot,
)
dagster_instance_for_k8s_run_launcher.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(external_pipeline),
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image()
def test_execute_on_celery_k8s_job_api_with_legacy_configmap_set( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace, dagit_url):
# Originally, jobs needed to include "dagster-pipeline-env" to pick up needed config when
# using the helm chart - it's no longer needed, but verify that nothing breaks if it's included
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_job_engine_config(
dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace,
include_dagster_pipeline_env=True,
),
)
run_id = launch_run_over_graphql(dagit_url,
run_config=run_config,
pipeline_name="demo_job_celery")
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == dagster_docker_image
def test_execute_subset_on_celery_k8s( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace):
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(),
"env_subset.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
pipeline_name = "demo_pipeline_celery"
run = create_run_for_test(
dagster_instance,
pipeline_name=pipeline_name,
run_config=run_config,
mode="default",
solids_to_execute={"count_letters"},
)
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(
get_test_project_external_pipeline(pipeline_name)),
)
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" %
run.run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_execute_on_celery_k8s_default( # pylint: disable=redefined-outer-name
dagster_docker_image,
dagster_instance,
helm_namespace,
dagit_url,
):
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
run_id = launch_run_over_graphql(dagit_url,
run_config=run_config,
pipeline_name="demo_pipeline_celery")
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == dagster_docker_image
def test_k8s_run_launcher_default(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
pods = DagsterKubernetesClient.production_client(
).core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher)
celery_pod_names = [
p.metadata.name for p in pods.items
if "celery-workers" in p.metadata.name
]
check.invariant(not celery_pod_names)
run_config = load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image(
)
def test_k8s_run_launcher_default(
dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher
):
# sanity check that we have a K8sRunLauncher
check.inst(dagster_instance_for_k8s_run_launcher.run_launcher, K8sRunLauncher)
pods = DagsterKubernetesClient.production_client().core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher
)
celery_pod_names = [p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name]
check.invariant(not celery_pod_names)
run_config = load_yaml_from_path(os.path.join(test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
tags = {"key": "value"}
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
tags=tags,
mode="default",
)
dagster_instance_for_k8s_run_launcher.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(get_test_project_external_pipeline(pipeline_name)),
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_failing_k8s_run_launcher(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
run_config = load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml"))
pipeline_name = "always_fail_pipeline"
run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format(
result)
event_records = dagster_instance_for_k8s_run_launcher.all_logs(run_id)
assert any(
["Op Exception Message" in str(event) for event in event_records])
def test_execute_on_celery_k8s( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace):
run_config = merge_dicts(
merge_yamls([
os.path.join(test_project_environments_path(), 'env.yaml'),
os.path.join(test_project_environments_path(), 'env_s3.yaml'),
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
pipeline_name = 'demo_pipeline_celery'
run = create_run_for_test(
dagster_instance,
pipeline_name=pipeline_name,
run_config=run_config,
mode='default',
)
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(
get_test_project_external_pipeline(pipeline_name)),
)
result = wait_for_job_and_get_raw_logs(job_name='dagster-run-%s' %
run.run_id,
namespace=helm_namespace)
assert 'PIPELINE_SUCCESS' in result, 'no match, result: {}'.format(result)
def test_execute_on_celery_k8s_retry_pipeline( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace):
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env_s3.yaml")
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
pipeline_name = "retry_pipeline"
run = create_run_for_test(
dagster_instance,
pipeline_name=pipeline_name,
run_config=run_config,
mode="default",
)
with get_test_project_external_pipeline(
pipeline_name) as external_pipeline:
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(external_pipeline),
)
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" %
run.run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(
result)
stats = dagster_instance.get_run_stats(run.run_id)
assert stats.steps_succeeded == 1
assert DagsterEventType.STEP_START in [
event.dagster_event.event_type
for event in dagster_instance.all_logs(run.run_id)
if event.is_dagster_event
]
assert DagsterEventType.STEP_UP_FOR_RETRY in [
event.dagster_event.event_type
for event in dagster_instance.all_logs(run.run_id)
if event.is_dagster_event
]
assert DagsterEventType.STEP_RESTARTED in [
event.dagster_event.event_type
for event in dagster_instance.all_logs(run.run_id)
if event.is_dagster_event
]
assert DagsterEventType.STEP_SUCCESS in [
event.dagster_event.event_type
for event in dagster_instance.all_logs(run.run_id)
if event.is_dagster_event
]
def test_memoization_on_celery_k8s( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace, dagit_url):
ephemeral_prefix = str(uuid.uuid4())
run_config = deep_merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env_s3.yaml")
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
run_config = deep_merge_dicts(
run_config,
{
"resources": {
"io_manager": {
"config": {
"s3_prefix": ephemeral_prefix
}
}
}
},
)
try:
run_ids = []
for _ in range(2):
run_id = launch_run_over_graphql(
dagit_url,
run_config=run_config,
pipeline_name="memoization_pipeline",
mode="celery",
)
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" %
run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(
result)
run_ids.append(run_id)
unmemoized_run_id = run_ids[0]
step_events = _get_step_events(
dagster_instance.all_logs(unmemoized_run_id))
assert len(step_events) == 4
memoized_run_id = run_ids[1]
step_events = _get_step_events(
dagster_instance.all_logs(memoized_run_id))
assert len(step_events) == 0
finally:
cleanup_memoized_results(define_memoization_pipeline(), "celery",
dagster_instance, run_config)
def test_execute_on_k8s_retry_pipeline( # pylint: disable=redefined-outer-name
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image,
dagit_url_for_k8s_run_launcher,
):
run_config = merge_dicts(
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
pipeline_name = "retry_pipeline"
run_id = launch_run_over_graphql(
dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name,
mode="k8s",
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
stats = dagster_instance_for_k8s_run_launcher.get_run_stats(run_id)
assert stats.steps_succeeded == 1
all_logs = dagster_instance_for_k8s_run_launcher.all_logs(run_id)
assert DagsterEventType.STEP_START in [
event.dagster_event.event_type for event in all_logs if event.is_dagster_event
]
assert DagsterEventType.STEP_UP_FOR_RETRY in [
event.dagster_event.event_type for event in all_logs if event.is_dagster_event
]
assert DagsterEventType.STEP_RESTARTED in [
event.dagster_event.event_type for event in all_logs if event.is_dagster_event
]
assert DagsterEventType.STEP_SUCCESS in [
event.dagster_event.event_type for event in all_logs if event.is_dagster_event
]
def test_execute_queeud_run_on_celery_k8s( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance_for_daemon,
helm_namespace_for_daemon):
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_engine_config(
dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace_for_daemon,
),
)
pipeline_name = "demo_pipeline_celery"
with get_test_project_external_pipeline(
pipeline_name) as external_pipeline:
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(
external_pipeline)
run = create_run_for_test(
dagster_instance_for_daemon,
pipeline_name=pipeline_name,
run_config=run_config,
mode="default",
external_pipeline_origin=reoriginated_pipeline.get_external_origin(
),
)
dagster_instance_for_daemon.submit_run(
run.run_id,
reoriginated_pipeline,
)
wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run.run_id,
namespace=helm_namespace_for_daemon)
logs = dagster_instance_for_daemon.all_logs(run.run_id)
assert_events_in_order(
logs,
[
"PIPELINE_ENQUEUED", "PIPELINE_DEQUEUED", "PIPELINE_STARTING",
"PIPELINE_SUCCESS"
],
)
def test_failing_k8s_run_launcher(dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher):
run_config = {"blah blah this is wrong": {}}
pipeline_name = "demo_pipeline"
with get_test_project_location_and_external_pipeline(pipeline_name) as (
location,
external_pipeline,
):
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=location.get_external_execution_plan(
external_pipeline,
{
"solids": {
"multiply_the_word": {
"config": {
"factor": 0
},
"inputs": {
"word": "..."
}
}
}
},
"default",
None,
None,
).execution_plan_snapshot,
)
dagster_instance_for_k8s_run_launcher.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(external_pipeline),
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format(
result)
event_records = dagster_instance_for_k8s_run_launcher.all_logs(
run.run_id)
assert any([
'Received unexpected config entry "blah blah this is wrong"'
in str(event) for event in event_records
])
assert any([
'Missing required config entry "solids"' in str(event)
for event in event_records
])
def _launch_executor_run(
run_config,
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
):
pipeline_name = "demo_k8s_executor_pipeline"
tags = {"key": "value"}
with get_test_project_external_pipeline_hierarchy(
dagster_instance_for_k8s_run_launcher, pipeline_name) as (
workspace,
location,
_repo,
external_pipeline,
):
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(
external_pipeline)
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
tags=tags,
mode="default",
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=location.get_external_execution_plan(
external_pipeline, run_config, "default", None,
None).execution_plan_snapshot,
external_pipeline_origin=reoriginated_pipeline.get_external_origin(
),
pipeline_code_origin=reoriginated_pipeline.get_python_origin(),
)
dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(
result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(
run.run_id)
assert updated_run.tags[
DOCKER_IMAGE_TAG] == get_test_project_docker_image()
events = dagster_instance_for_k8s_run_launcher.all_logs(run.run_id)
assert (len([
event for event in events
if ("Executing step" in event.message
and "in Kubernetes job" in event.message)
]) == 2)
def test_k8s_executor_resource_requirements(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image,
dagit_url_for_k8s_run_launcher,
):
# sanity check that we have a K8sRunLauncher
pods = DagsterKubernetesClient.production_client(
).core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher)
celery_pod_names = [
p.metadata.name for p in pods.items
if "celery-workers" in p.metadata.name
]
check.invariant(not celery_pod_names)
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
pipeline_name = "resources_limit_pipeline"
run_id = launch_run_over_graphql(
dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name,
mode="k8s",
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image(
)
def test_k8s_run_launcher_image_from_origin(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
# Like the previous test, but the executor doesn't supply an image - it's pulled
# from the origin on the run instead
pods = DagsterKubernetesClient.production_client(
).core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher)
celery_pod_names = [
p.metadata.name for p in pods.items
if "celery-workers" in p.metadata.name
]
check.invariant(not celery_pod_names)
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml")),
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
pipeline_name = "demo_k8s_executor_pipeline"
run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image(
)
def test_k8s_run_launcher(dagster_instance, helm_namespace):
run_config = load_yaml_from_path(os.path.join(test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
run = create_run_for_test(
dagster_instance, pipeline_name=pipeline_name, run_config=run_config, mode="default",
)
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(get_test_project_external_pipeline(pipeline_name)),
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_k8s_run_launcher_default(
dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher
):
# sanity check that we have a K8sRunLauncher
check.inst(dagster_instance_for_k8s_run_launcher.run_launcher, K8sRunLauncher)
pods = DagsterKubernetesClient.production_client().core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher
)
celery_pod_names = [p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name]
check.invariant(not celery_pod_names)
run_config = load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
tags = {"key": "value"}
with get_test_project_external_pipeline_hierarchy(
dagster_instance_for_k8s_run_launcher, pipeline_name
) as (
workspace,
location,
_repo,
external_pipeline,
):
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(external_pipeline)
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
tags=tags,
mode="default",
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=location.get_external_execution_plan(
external_pipeline, run_config, "default", None, None
).execution_plan_snapshot,
external_pipeline_origin=reoriginated_pipeline.get_external_origin(),
pipeline_code_origin=reoriginated_pipeline.get_python_origin(),
)
dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image()
def test_execute_on_celery_k8s_with_resource_requirements( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace, dagit_url):
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
run_id = launch_run_over_graphql(dagit_url,
run_config=run_config,
pipeline_name="resources_limit_pipeline")
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_execute_on_celery_k8s_image_from_origin( # pylint: disable=redefined-outer-name
dagster_docker_image, dagster_instance, helm_namespace):
# Like the previous test, but the image is included in the pipeline origin
# rather than in the executor config
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_engine_config(dagster_docker_image=None,
job_namespace=helm_namespace),
)
pipeline_name = "demo_pipeline_celery"
with get_test_project_workspace_and_external_pipeline(
dagster_instance, pipeline_name,
container_image=dagster_docker_image) as (workspace,
external_pipeline):
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(
external_pipeline, container_image=dagster_docker_image)
run = create_run_for_test(
dagster_instance,
pipeline_name=pipeline_name,
run_config=run_config,
mode="default",
external_pipeline_origin=reoriginated_pipeline.get_external_origin(
),
pipeline_code_origin=reoriginated_pipeline.get_python_origin(),
)
dagster_instance.launch_run(run.run_id, workspace)
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" %
run.run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(
result)
updated_run = dagster_instance.get_run_by_id(run.run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == dagster_docker_image
def test_volume_mounts(dagster_docker_image, dagster_instance, helm_namespace,
dagit_url):
run_config = deep_merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env_s3.yaml")
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
run_id = launch_run_over_graphql(
dagit_url,
run_config=run_config,
pipeline_name="volume_mount_pipeline",
mode="celery",
)
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_failing_k8s_run_launcher(dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher):
run_config = {"blah blah this is wrong": {}}
pipeline_name = "demo_pipeline"
with get_test_project_workspace_and_external_pipeline(
dagster_instance_for_k8s_run_launcher, pipeline_name) as (
workspace,
external_pipeline,
):
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(
external_pipeline)
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
external_pipeline_origin=reoriginated_pipeline.get_external_origin(
),
pipeline_code_origin=reoriginated_pipeline.get_python_origin(),
)
dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format(
result)
event_records = dagster_instance_for_k8s_run_launcher.all_logs(
run.run_id)
assert any([
'Received unexpected config entry "blah blah this is wrong"'
in str(event) for event in event_records
])
assert any([
'Missing required config entry "solids"' in str(event)
for event in event_records
])
def test_k8s_run_launcher_secret_from_deployment(
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
# This run_config requires that WORD_FACTOR be set on both the user code deployment
# and the run launcher. It will only work if secrets are propagated from the deployment
# to the run launcher, since TEST_DEPLOYMENT_SECRET_NAME is only set on the user code
# deployment but not on the run launcher config.
run_config = load_yaml_from_path(
os.path.join(get_test_project_environments_path(),
"env_config_from_secrets.yaml"))
pipeline_name = "demo_pipeline"
run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_execute_subset_on_celery_k8s( # pylint: disable=redefined-outer-name
dagster_docker_image, helm_namespace, dagit_url):
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(),
"env_subset.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
get_celery_engine_config(dagster_docker_image=dagster_docker_image,
job_namespace=helm_namespace),
)
run_id = launch_run_over_graphql(
dagit_url,
run_config=run_config,
pipeline_name="demo_pipeline_celery",
solid_selection=["count_letters"],
)
result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id,
namespace=helm_namespace)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_k8s_run_launcher_default(dagster_instance, helm_namespace):
run_config = load_yaml_from_path(
os.path.join(test_project_environments_path(), 'env.yaml'))
pipeline_name = 'demo_pipeline'
tags = {'key': 'value'}
run = create_run_for_test(
dagster_instance,
pipeline_name=pipeline_name,
run_config=run_config,
tags=tags,
mode='default',
)
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(
get_test_project_external_pipeline(pipeline_name)),
)
result = wait_for_job_and_get_raw_logs(job_name='dagster-run-%s' %
run.run_id,
namespace=helm_namespace)
assert 'PIPELINE_SUCCESS' in result, 'no match, result: {}'.format(result)
