def wait_for_job_ready(job_name, namespace):
"""Wait for a dagster-k8s job to be ready
"""
check.str_param(job_name, "job_name")
check.str_param(namespace, "namespace")
wait_for_job(job_name=job_name, namespace=namespace)
def wait_for_job_ready(job_name, namespace):
'''Wait for a dagster-k8s job to be ready
'''
check.str_param(job_name, 'job_name')
check.str_param(namespace, 'namespace')
wait_for_job(job_name=job_name, namespace=namespace)
def test_k8s_run_launcher_terminate(
dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher
):
pipeline_name = "slow_pipeline"
tags = {"key": "value"}
with get_test_project_external_pipeline_hierarchy(
dagster_instance_for_k8s_run_launcher, pipeline_name
) as (
workspace,
location,
_repo,
external_pipeline,
):
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(external_pipeline)
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=None,
tags=tags,
mode="default",
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=location.get_external_execution_plan(
external_pipeline, {}, "default", None, None
).execution_plan_snapshot,
external_pipeline_origin=reoriginated_pipeline.get_external_origin(),
pipeline_code_origin=reoriginated_pipeline.get_python_origin(),
)
dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace)
wait_for_job(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher
)
timeout = datetime.timedelta(0, 30)
start_time = datetime.datetime.now()
while datetime.datetime.now() < start_time + timeout:
if dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(run_id=run.run_id):
break
time.sleep(5)
assert dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(run_id=run.run_id)
assert dagster_instance_for_k8s_run_launcher.run_launcher.terminate(run_id=run.run_id)
start_time = datetime.datetime.now()
pipeline_run = None
while datetime.datetime.now() < start_time + timeout:
pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id)
if pipeline_run.status == PipelineRunStatus.CANCELED:
break
time.sleep(5)
assert pipeline_run.status == PipelineRunStatus.CANCELED
assert not dagster_instance_for_k8s_run_launcher.run_launcher.terminate(run_id=run.run_id)
def test_k8s_run_launcher_terminate(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image,
dagit_url_for_k8s_run_launcher,
):
pipeline_name = "slow_pipeline"
run_config = merge_dicts(
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
run_id = launch_run_over_graphql(
dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name,
mode="k8s",
)
wait_for_job(job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher)
timeout = datetime.timedelta(0, 30)
start_time = datetime.datetime.now()
while True:
assert datetime.datetime.now() < start_time + timeout, "Timed out waiting for can_terminate"
if can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id):
break
time.sleep(5)
terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id=run_id)
start_time = datetime.datetime.now()
pipeline_run = None
while True:
assert datetime.datetime.now() < start_time + timeout, "Timed out waiting for termination"
pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
if pipeline_run.status == PipelineRunStatus.CANCELED:
break
time.sleep(5)
# useful to have logs here, because the worker pods get deleted
print(dagster_instance_for_k8s_run_launcher.all_logs(run_id)) # pylint: disable=print-call
assert pipeline_run.status == PipelineRunStatus.CANCELED
assert not can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id)
def test_k8s_run_launcher_terminate(dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher):
pipeline_name = "slow_pipeline"
tags = {"key": "value"}
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=None,
tags=tags,
mode="default",
)
with get_test_project_external_pipeline(
pipeline_name) as external_pipeline:
dagster_instance_for_k8s_run_launcher.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(external_pipeline),
)
wait_for_job(job_name="dagster-run-%s" % run.run_id,
namespace=helm_namespace_for_k8s_run_launcher)
timeout = datetime.timedelta(0, 30)
start_time = datetime.datetime.now()
while datetime.datetime.now() < start_time + timeout:
if dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(
run_id=run.run_id):
break
time.sleep(5)
assert dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(
run_id=run.run_id)
assert dagster_instance_for_k8s_run_launcher.run_launcher.terminate(
run_id=run.run_id)
start_time = datetime.datetime.now()
pipeline_run = None
while datetime.datetime.now() < start_time + timeout:
pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(
run.run_id)
if pipeline_run.status == PipelineRunStatus.CANCELED:
break
time.sleep(5)
assert pipeline_run.status == PipelineRunStatus.CANCELED
assert not dagster_instance_for_k8s_run_launcher.run_launcher.terminate(
run_id=run.run_id)
def test_k8s_run_launcher_terminate(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
pipeline_name = "slow_pipeline"
run_config = load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml"))
run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name)
wait_for_job(job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
timeout = datetime.timedelta(0, 30)
start_time = datetime.datetime.now()
while True:
assert datetime.datetime.now(
) < start_time + timeout, "Timed out waiting for can_terminate"
if can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_id):
break
time.sleep(5)
terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id=run_id)
start_time = datetime.datetime.now()
pipeline_run = None
while True:
assert datetime.datetime.now(
) < start_time + timeout, "Timed out waiting for termination"
pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(
run_id)
if pipeline_run.status == PipelineRunStatus.CANCELED:
break
time.sleep(5)
assert pipeline_run.status == PipelineRunStatus.CANCELED
assert not can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_id)
def test_k8s_run_launcher_terminate(dagster_instance, helm_namespace):
pipeline_name = 'slow_pipeline'
tags = {'key': 'value'}
run = create_run_for_test(
dagster_instance,
pipeline_name=pipeline_name,
run_config=None,
tags=tags,
mode='default',
)
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(
get_test_project_external_pipeline(pipeline_name)),
)
wait_for_job(job_name='dagster-run-%s' % run.run_id,
namespace=helm_namespace)
timeout = datetime.timedelta(0, 30)
start_time = datetime.datetime.now()
while datetime.datetime.now() < start_time + timeout:
if dagster_instance.run_launcher.can_terminate(run_id=run.run_id):
break
time.sleep(5)
assert dagster_instance.run_launcher.can_terminate(run_id=run.run_id)
assert dagster_instance.run_launcher.terminate(run_id=run.run_id)
start_time = datetime.datetime.now()
pipeline_run = None
while datetime.datetime.now() < start_time + timeout:
pipeline_run = dagster_instance.get_run_by_id(run.run_id)
if pipeline_run.status == PipelineRunStatus.FAILURE:
break
time.sleep(5)
assert pipeline_run.status == PipelineRunStatus.FAILURE
assert not dagster_instance.run_launcher.terminate(run_id=run.run_id)
def test_k8s_run_launcher_terminate(
dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagster_docker_image
):
pipeline_name = "slow_pipeline"
tags = {"key": "value"}
run_config = merge_dicts(
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
"env_config_maps": ["dagster-pipeline-env"]
+ ([TEST_AWS_CONFIGMAP_NAME] if not IS_BUILDKITE else []),
}
}
},
},
)
with get_test_project_external_pipeline_hierarchy(
dagster_instance_for_k8s_run_launcher, pipeline_name
) as (
workspace,
location,
_repo,
external_pipeline,
):
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(external_pipeline)
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
tags=tags,
mode="k8s",
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=location.get_external_execution_plan(
external_pipeline, run_config, "k8s", None, None
).execution_plan_snapshot,
external_pipeline_origin=reoriginated_pipeline.get_external_origin(),
pipeline_code_origin=reoriginated_pipeline.get_python_origin(),
)
dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace)
wait_for_job(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher
)
timeout = datetime.timedelta(0, 30)
start_time = datetime.datetime.now()
while datetime.datetime.now() < start_time + timeout:
if dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(run_id=run.run_id):
break
time.sleep(5)
assert dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(run_id=run.run_id)
assert dagster_instance_for_k8s_run_launcher.run_launcher.terminate(run_id=run.run_id)
start_time = datetime.datetime.now()
pipeline_run = None
while datetime.datetime.now() < start_time + timeout:
pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id)
if pipeline_run.status == PipelineRunStatus.CANCELED:
break
time.sleep(5)
# useful to have logs here, because the worker pods get deleted
print( # pylint: disable=print-call
dagster_instance_for_k8s_run_launcher.all_logs(run.run_id)
)
assert pipeline_run.status == PipelineRunStatus.CANCELED
assert not dagster_instance_for_k8s_run_launcher.run_launcher.terminate(run_id=run.run_id)
