def wait_for_job_ready(job_name, namespace): """Wait for a dagster-k8s job to be ready """ check.str_param(job_name, "job_name") check.str_param(namespace, "namespace") wait_for_job(job_name=job_name, namespace=namespace)
def wait_for_job_ready(job_name, namespace): '''Wait for a dagster-k8s job to be ready ''' check.str_param(job_name, 'job_name') check.str_param(namespace, 'namespace') wait_for_job(job_name=job_name, namespace=namespace)
def test_k8s_run_launcher_terminate( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher ): pipeline_name = "slow_pipeline" tags = {"key": "value"} with get_test_project_external_pipeline_hierarchy( dagster_instance_for_k8s_run_launcher, pipeline_name ) as ( workspace, location, _repo, external_pipeline, ): reoriginated_pipeline = ReOriginatedExternalPipelineForTest(external_pipeline) run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=None, tags=tags, mode="default", pipeline_snapshot=external_pipeline.pipeline_snapshot, execution_plan_snapshot=location.get_external_execution_plan( external_pipeline, {}, "default", None, None ).execution_plan_snapshot, external_pipeline_origin=reoriginated_pipeline.get_external_origin(), pipeline_code_origin=reoriginated_pipeline.get_python_origin(), ) dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace) wait_for_job( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher ) timeout = datetime.timedelta(0, 30) start_time = datetime.datetime.now() while datetime.datetime.now() < start_time + timeout: if dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(run_id=run.run_id): break time.sleep(5) assert dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(run_id=run.run_id) assert dagster_instance_for_k8s_run_launcher.run_launcher.terminate(run_id=run.run_id) start_time = datetime.datetime.now() pipeline_run = None while datetime.datetime.now() < start_time + timeout: pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id) if pipeline_run.status == PipelineRunStatus.CANCELED: break time.sleep(5) assert pipeline_run.status == PipelineRunStatus.CANCELED assert not dagster_instance_for_k8s_run_launcher.run_launcher.terminate(run_id=run.run_id)
def test_k8s_run_launcher_terminate( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagster_docker_image, dagit_url_for_k8s_run_launcher, ): pipeline_name = "slow_pipeline" run_config = merge_dicts( load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")), { "execution": { "k8s": { "config": { "job_namespace": helm_namespace_for_k8s_run_launcher, "job_image": dagster_docker_image, "image_pull_policy": image_pull_policy(), } } }, }, ) run_id = launch_run_over_graphql( dagit_url_for_k8s_run_launcher, run_config=run_config, pipeline_name=pipeline_name, mode="k8s", ) wait_for_job(job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher) timeout = datetime.timedelta(0, 30) start_time = datetime.datetime.now() while True: assert datetime.datetime.now() < start_time + timeout, "Timed out waiting for can_terminate" if can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id): break time.sleep(5) terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id=run_id) start_time = datetime.datetime.now() pipeline_run = None while True: assert datetime.datetime.now() < start_time + timeout, "Timed out waiting for termination" pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id) if pipeline_run.status == PipelineRunStatus.CANCELED: break time.sleep(5) # useful to have logs here, because the worker pods get deleted print(dagster_instance_for_k8s_run_launcher.all_logs(run_id)) # pylint: disable=print-call assert pipeline_run.status == PipelineRunStatus.CANCELED assert not can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id)
def test_k8s_run_launcher_terminate(dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher): pipeline_name = "slow_pipeline" tags = {"key": "value"} run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=None, tags=tags, mode="default", ) with get_test_project_external_pipeline( pipeline_name) as external_pipeline: dagster_instance_for_k8s_run_launcher.launch_run( run.run_id, ReOriginatedExternalPipelineForTest(external_pipeline), ) wait_for_job(job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher) timeout = datetime.timedelta(0, 30) start_time = datetime.datetime.now() while datetime.datetime.now() < start_time + timeout: if dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate( run_id=run.run_id): break time.sleep(5) assert dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate( run_id=run.run_id) assert dagster_instance_for_k8s_run_launcher.run_launcher.terminate( run_id=run.run_id) start_time = datetime.datetime.now() pipeline_run = None while datetime.datetime.now() < start_time + timeout: pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id( run.run_id) if pipeline_run.status == PipelineRunStatus.CANCELED: break time.sleep(5) assert pipeline_run.status == PipelineRunStatus.CANCELED assert not dagster_instance_for_k8s_run_launcher.run_launcher.terminate( run_id=run.run_id)
def test_k8s_run_launcher_terminate( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagit_url_for_k8s_run_launcher, ): pipeline_name = "slow_pipeline" run_config = load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env_s3.yaml")) run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher, run_config=run_config, pipeline_name=pipeline_name) wait_for_job(job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher) timeout = datetime.timedelta(0, 30) start_time = datetime.datetime.now() while True: assert datetime.datetime.now( ) < start_time + timeout, "Timed out waiting for can_terminate" if can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id): break time.sleep(5) terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id=run_id) start_time = datetime.datetime.now() pipeline_run = None while True: assert datetime.datetime.now( ) < start_time + timeout, "Timed out waiting for termination" pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id( run_id) if pipeline_run.status == PipelineRunStatus.CANCELED: break time.sleep(5) assert pipeline_run.status == PipelineRunStatus.CANCELED assert not can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id)
def test_k8s_run_launcher_terminate(dagster_instance, helm_namespace): pipeline_name = 'slow_pipeline' tags = {'key': 'value'} run = create_run_for_test( dagster_instance, pipeline_name=pipeline_name, run_config=None, tags=tags, mode='default', ) dagster_instance.launch_run( run.run_id, ReOriginatedExternalPipelineForTest( get_test_project_external_pipeline(pipeline_name)), ) wait_for_job(job_name='dagster-run-%s' % run.run_id, namespace=helm_namespace) timeout = datetime.timedelta(0, 30) start_time = datetime.datetime.now() while datetime.datetime.now() < start_time + timeout: if dagster_instance.run_launcher.can_terminate(run_id=run.run_id): break time.sleep(5) assert dagster_instance.run_launcher.can_terminate(run_id=run.run_id) assert dagster_instance.run_launcher.terminate(run_id=run.run_id) start_time = datetime.datetime.now() pipeline_run = None while datetime.datetime.now() < start_time + timeout: pipeline_run = dagster_instance.get_run_by_id(run.run_id) if pipeline_run.status == PipelineRunStatus.FAILURE: break time.sleep(5) assert pipeline_run.status == PipelineRunStatus.FAILURE assert not dagster_instance.run_launcher.terminate(run_id=run.run_id)
def test_k8s_run_launcher_terminate( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagster_docker_image ): pipeline_name = "slow_pipeline" tags = {"key": "value"} run_config = merge_dicts( load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")), { "execution": { "k8s": { "config": { "job_namespace": helm_namespace_for_k8s_run_launcher, "job_image": dagster_docker_image, "image_pull_policy": image_pull_policy(), "env_config_maps": ["dagster-pipeline-env"] + ([TEST_AWS_CONFIGMAP_NAME] if not IS_BUILDKITE else []), } } }, }, ) with get_test_project_external_pipeline_hierarchy( dagster_instance_for_k8s_run_launcher, pipeline_name ) as ( workspace, location, _repo, external_pipeline, ): reoriginated_pipeline = ReOriginatedExternalPipelineForTest(external_pipeline) run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=run_config, tags=tags, mode="k8s", pipeline_snapshot=external_pipeline.pipeline_snapshot, execution_plan_snapshot=location.get_external_execution_plan( external_pipeline, run_config, "k8s", None, None ).execution_plan_snapshot, external_pipeline_origin=reoriginated_pipeline.get_external_origin(), pipeline_code_origin=reoriginated_pipeline.get_python_origin(), ) dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace) wait_for_job( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher ) timeout = datetime.timedelta(0, 30) start_time = datetime.datetime.now() while datetime.datetime.now() < start_time + timeout: if dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(run_id=run.run_id): break time.sleep(5) assert dagster_instance_for_k8s_run_launcher.run_launcher.can_terminate(run_id=run.run_id) assert dagster_instance_for_k8s_run_launcher.run_launcher.terminate(run_id=run.run_id) start_time = datetime.datetime.now() pipeline_run = None while datetime.datetime.now() < start_time + timeout: pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id) if pipeline_run.status == PipelineRunStatus.CANCELED: break time.sleep(5) # useful to have logs here, because the worker pods get deleted print( # pylint: disable=print-call dagster_instance_for_k8s_run_launcher.all_logs(run.run_id) ) assert pipeline_run.status == PipelineRunStatus.CANCELED assert not dagster_instance_for_k8s_run_launcher.run_launcher.terminate(run_id=run.run_id)