def test_execute_queued_run_on_celery_k8s( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance_for_daemon, helm_namespace_for_daemon, dagit_url_for_daemon, ): run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_engine_config( dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace_for_daemon, ), ) run_id = launch_run_over_graphql(dagit_url_for_daemon, run_config=run_config, pipeline_name="demo_pipeline_celery") wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_daemon) logs = dagster_instance_for_daemon.all_logs(run_id) assert_events_in_order( logs, [ "PIPELINE_ENQUEUED", "PIPELINE_DEQUEUED", "PIPELINE_STARTING", "PIPELINE_SUCCESS" ], )
def test_execute_on_celery_k8s_default( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace): run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) pipeline_name = "demo_pipeline_celery" run = create_run_for_test( dagster_instance, pipeline_name=pipeline_name, run_config=run_config, mode="default", ) with get_test_project_external_pipeline( pipeline_name) as external_pipeline: dagster_instance.launch_run( run.run_id, ReOriginatedExternalPipelineForTest(external_pipeline), ) result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format( result) updated_run = dagster_instance.get_run_by_id(run.run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == dagster_docker_image
def test_k8s_run_launcher(dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher): run_config = load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env.yaml")) pipeline_name = "demo_pipeline" with get_test_project_workspace_and_external_pipeline( dagster_instance_for_k8s_run_launcher, pipeline_name) as ( workspace, external_pipeline, ): reoriginated_pipeline = ReOriginatedExternalPipelineForTest( external_pipeline) run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=run_config, mode="default", external_pipeline_origin=reoriginated_pipeline.get_external_origin( ), pipeline_code_origin=reoriginated_pipeline.get_python_origin(), ) dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format( result)
def test_failing_k8s_run_launcher(dagster_instance, helm_namespace): run_config = {"blah blah this is wrong": {}} pipeline_name = "demo_pipeline" run = create_run_for_test(dagster_instance, pipeline_name=pipeline_name, run_config=run_config) dagster_instance.launch_run( run.run_id, ReOriginatedExternalPipelineForTest( get_test_project_external_pipeline(pipeline_name)), ) result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format( result) event_records = dagster_instance.all_logs(run.run_id) assert any([ 'Undefined field "blah blah this is wrong"' in str(event) for event in event_records ]) assert any([ 'Missing required field "solids"' in str(event) for event in event_records ])
def _launch_executor_run( dagit_url, run_config, dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, pipeline_name="demo_k8s_executor_pipeline", num_steps=2, mode="default", ): run_id = launch_run_over_graphql(dagit_url, run_config=run_config, pipeline_name=pipeline_name, mode=mode) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image( ) events = dagster_instance_for_k8s_run_launcher.all_logs(run_id) assert len(_get_step_execution_events(events)) == num_steps return run_id
def test_k8s_run_launcher_default( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagster_docker_image ): # sanity check that we have a K8sRunLauncher check.inst(dagster_instance_for_k8s_run_launcher.run_launcher, K8sRunLauncher) pods = DagsterKubernetesClient.production_client().core_api.list_namespaced_pod( namespace=helm_namespace_for_k8s_run_launcher ) celery_pod_names = [p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name] check.invariant(not celery_pod_names) run_config = merge_dicts( load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env.yaml")), load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")), { "execution": { "k8s": { "config": { "job_namespace": helm_namespace_for_k8s_run_launcher, "job_image": dagster_docker_image, "image_pull_policy": image_pull_policy(), "env_config_maps": ["dagster-pipeline-env"] + ([TEST_AWS_CONFIGMAP_NAME] if not IS_BUILDKITE else []), } } }, }, ) pipeline_name = "demo_k8s_executor_pipeline" tags = {"key": "value"} with get_test_project_location_and_external_pipeline(pipeline_name) as ( location, external_pipeline, ): run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=run_config, tags=tags, mode="default", pipeline_snapshot=external_pipeline.pipeline_snapshot, execution_plan_snapshot=location.get_external_execution_plan( external_pipeline, run_config, "default", None, None ).execution_plan_snapshot, ) dagster_instance_for_k8s_run_launcher.launch_run( run.run_id, ReOriginatedExternalPipelineForTest(external_pipeline), ) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher ) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image()
def test_execute_on_celery_k8s_job_api_with_legacy_configmap_set( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace, dagit_url): # Originally, jobs needed to include "dagster-pipeline-env" to pick up needed config when # using the helm chart - it's no longer needed, but verify that nothing breaks if it's included run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_job_engine_config( dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace, include_dagster_pipeline_env=True, ), ) run_id = launch_run_over_graphql(dagit_url, run_config=run_config, pipeline_name="demo_job_celery") result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) updated_run = dagster_instance.get_run_by_id(run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == dagster_docker_image
def test_execute_subset_on_celery_k8s( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace): run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env_subset.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) pipeline_name = "demo_pipeline_celery" run = create_run_for_test( dagster_instance, pipeline_name=pipeline_name, run_config=run_config, mode="default", solids_to_execute={"count_letters"}, ) dagster_instance.launch_run( run.run_id, ReOriginatedExternalPipelineForTest( get_test_project_external_pipeline(pipeline_name)), ) result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_execute_on_celery_k8s_default( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace, dagit_url, ): run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) run_id = launch_run_over_graphql(dagit_url, run_config=run_config, pipeline_name="demo_pipeline_celery") result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) updated_run = dagster_instance.get_run_by_id(run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == dagster_docker_image
def test_k8s_run_launcher_default( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagit_url_for_k8s_run_launcher, ): pods = DagsterKubernetesClient.production_client( ).core_api.list_namespaced_pod( namespace=helm_namespace_for_k8s_run_launcher) celery_pod_names = [ p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name ] check.invariant(not celery_pod_names) run_config = load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env.yaml")) pipeline_name = "demo_pipeline" run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher, run_config=run_config, pipeline_name=pipeline_name) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image( )
def test_k8s_run_launcher_default( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher ): # sanity check that we have a K8sRunLauncher check.inst(dagster_instance_for_k8s_run_launcher.run_launcher, K8sRunLauncher) pods = DagsterKubernetesClient.production_client().core_api.list_namespaced_pod( namespace=helm_namespace_for_k8s_run_launcher ) celery_pod_names = [p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name] check.invariant(not celery_pod_names) run_config = load_yaml_from_path(os.path.join(test_project_environments_path(), "env.yaml")) pipeline_name = "demo_pipeline" tags = {"key": "value"} run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=run_config, tags=tags, mode="default", ) dagster_instance_for_k8s_run_launcher.launch_run( run.run_id, ReOriginatedExternalPipelineForTest(get_test_project_external_pipeline(pipeline_name)), ) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher ) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_failing_k8s_run_launcher( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagit_url_for_k8s_run_launcher, ): run_config = load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env.yaml")) pipeline_name = "always_fail_pipeline" run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher, run_config=run_config, pipeline_name=pipeline_name) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format( result) event_records = dagster_instance_for_k8s_run_launcher.all_logs(run_id) assert any( ["Op Exception Message" in str(event) for event in event_records])
def test_execute_on_celery_k8s( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace): run_config = merge_dicts( merge_yamls([ os.path.join(test_project_environments_path(), 'env.yaml'), os.path.join(test_project_environments_path(), 'env_s3.yaml'), ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) pipeline_name = 'demo_pipeline_celery' run = create_run_for_test( dagster_instance, pipeline_name=pipeline_name, run_config=run_config, mode='default', ) dagster_instance.launch_run( run.run_id, ReOriginatedExternalPipelineForTest( get_test_project_external_pipeline(pipeline_name)), ) result = wait_for_job_and_get_raw_logs(job_name='dagster-run-%s' % run.run_id, namespace=helm_namespace) assert 'PIPELINE_SUCCESS' in result, 'no match, result: {}'.format(result)
def test_execute_on_celery_k8s_retry_pipeline( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace): run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env_s3.yaml") ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) pipeline_name = "retry_pipeline" run = create_run_for_test( dagster_instance, pipeline_name=pipeline_name, run_config=run_config, mode="default", ) with get_test_project_external_pipeline( pipeline_name) as external_pipeline: dagster_instance.launch_run( run.run_id, ReOriginatedExternalPipelineForTest(external_pipeline), ) result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format( result) stats = dagster_instance.get_run_stats(run.run_id) assert stats.steps_succeeded == 1 assert DagsterEventType.STEP_START in [ event.dagster_event.event_type for event in dagster_instance.all_logs(run.run_id) if event.is_dagster_event ] assert DagsterEventType.STEP_UP_FOR_RETRY in [ event.dagster_event.event_type for event in dagster_instance.all_logs(run.run_id) if event.is_dagster_event ] assert DagsterEventType.STEP_RESTARTED in [ event.dagster_event.event_type for event in dagster_instance.all_logs(run.run_id) if event.is_dagster_event ] assert DagsterEventType.STEP_SUCCESS in [ event.dagster_event.event_type for event in dagster_instance.all_logs(run.run_id) if event.is_dagster_event ]
def test_memoization_on_celery_k8s( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace, dagit_url): ephemeral_prefix = str(uuid.uuid4()) run_config = deep_merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env_s3.yaml") ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) run_config = deep_merge_dicts( run_config, { "resources": { "io_manager": { "config": { "s3_prefix": ephemeral_prefix } } } }, ) try: run_ids = [] for _ in range(2): run_id = launch_run_over_graphql( dagit_url, run_config=run_config, pipeline_name="memoization_pipeline", mode="celery", ) result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format( result) run_ids.append(run_id) unmemoized_run_id = run_ids[0] step_events = _get_step_events( dagster_instance.all_logs(unmemoized_run_id)) assert len(step_events) == 4 memoized_run_id = run_ids[1] step_events = _get_step_events( dagster_instance.all_logs(memoized_run_id)) assert len(step_events) == 0 finally: cleanup_memoized_results(define_memoization_pipeline(), "celery", dagster_instance, run_config)
def test_execute_on_k8s_retry_pipeline( # pylint: disable=redefined-outer-name dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagster_docker_image, dagit_url_for_k8s_run_launcher, ): run_config = merge_dicts( load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")), { "execution": { "k8s": { "config": { "job_namespace": helm_namespace_for_k8s_run_launcher, "job_image": dagster_docker_image, "image_pull_policy": image_pull_policy(), } } }, }, ) pipeline_name = "retry_pipeline" run_id = launch_run_over_graphql( dagit_url_for_k8s_run_launcher, run_config=run_config, pipeline_name=pipeline_name, mode="k8s", ) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher ) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) stats = dagster_instance_for_k8s_run_launcher.get_run_stats(run_id) assert stats.steps_succeeded == 1 all_logs = dagster_instance_for_k8s_run_launcher.all_logs(run_id) assert DagsterEventType.STEP_START in [ event.dagster_event.event_type for event in all_logs if event.is_dagster_event ] assert DagsterEventType.STEP_UP_FOR_RETRY in [ event.dagster_event.event_type for event in all_logs if event.is_dagster_event ] assert DagsterEventType.STEP_RESTARTED in [ event.dagster_event.event_type for event in all_logs if event.is_dagster_event ] assert DagsterEventType.STEP_SUCCESS in [ event.dagster_event.event_type for event in all_logs if event.is_dagster_event ]
def test_execute_queeud_run_on_celery_k8s( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance_for_daemon, helm_namespace_for_daemon): run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_engine_config( dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace_for_daemon, ), ) pipeline_name = "demo_pipeline_celery" with get_test_project_external_pipeline( pipeline_name) as external_pipeline: reoriginated_pipeline = ReOriginatedExternalPipelineForTest( external_pipeline) run = create_run_for_test( dagster_instance_for_daemon, pipeline_name=pipeline_name, run_config=run_config, mode="default", external_pipeline_origin=reoriginated_pipeline.get_external_origin( ), ) dagster_instance_for_daemon.submit_run( run.run_id, reoriginated_pipeline, ) wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_daemon) logs = dagster_instance_for_daemon.all_logs(run.run_id) assert_events_in_order( logs, [ "PIPELINE_ENQUEUED", "PIPELINE_DEQUEUED", "PIPELINE_STARTING", "PIPELINE_SUCCESS" ], )
def test_failing_k8s_run_launcher(dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher): run_config = {"blah blah this is wrong": {}} pipeline_name = "demo_pipeline" with get_test_project_location_and_external_pipeline(pipeline_name) as ( location, external_pipeline, ): run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=run_config, pipeline_snapshot=external_pipeline.pipeline_snapshot, execution_plan_snapshot=location.get_external_execution_plan( external_pipeline, { "solids": { "multiply_the_word": { "config": { "factor": 0 }, "inputs": { "word": "..." } } } }, "default", None, None, ).execution_plan_snapshot, ) dagster_instance_for_k8s_run_launcher.launch_run( run.run_id, ReOriginatedExternalPipelineForTest(external_pipeline), ) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format( result) event_records = dagster_instance_for_k8s_run_launcher.all_logs( run.run_id) assert any([ 'Received unexpected config entry "blah blah this is wrong"' in str(event) for event in event_records ]) assert any([ 'Missing required config entry "solids"' in str(event) for event in event_records ])
def _launch_executor_run( run_config, dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, ): pipeline_name = "demo_k8s_executor_pipeline" tags = {"key": "value"} with get_test_project_external_pipeline_hierarchy( dagster_instance_for_k8s_run_launcher, pipeline_name) as ( workspace, location, _repo, external_pipeline, ): reoriginated_pipeline = ReOriginatedExternalPipelineForTest( external_pipeline) run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=run_config, tags=tags, mode="default", pipeline_snapshot=external_pipeline.pipeline_snapshot, execution_plan_snapshot=location.get_external_execution_plan( external_pipeline, run_config, "default", None, None).execution_plan_snapshot, external_pipeline_origin=reoriginated_pipeline.get_external_origin( ), pipeline_code_origin=reoriginated_pipeline.get_python_origin(), ) dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format( result) updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id( run.run_id) assert updated_run.tags[ DOCKER_IMAGE_TAG] == get_test_project_docker_image() events = dagster_instance_for_k8s_run_launcher.all_logs(run.run_id) assert (len([ event for event in events if ("Executing step" in event.message and "in Kubernetes job" in event.message) ]) == 2)
def test_k8s_executor_resource_requirements( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagster_docker_image, dagit_url_for_k8s_run_launcher, ): # sanity check that we have a K8sRunLauncher pods = DagsterKubernetesClient.production_client( ).core_api.list_namespaced_pod( namespace=helm_namespace_for_k8s_run_launcher) celery_pod_names = [ p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name ] check.invariant(not celery_pod_names) run_config = merge_dicts( load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env_s3.yaml")), { "execution": { "k8s": { "config": { "job_namespace": helm_namespace_for_k8s_run_launcher, "job_image": dagster_docker_image, "image_pull_policy": image_pull_policy(), } } }, }, ) pipeline_name = "resources_limit_pipeline" run_id = launch_run_over_graphql( dagit_url_for_k8s_run_launcher, run_config=run_config, pipeline_name=pipeline_name, mode="k8s", ) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image( )
def test_k8s_run_launcher_image_from_origin( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagit_url_for_k8s_run_launcher, ): # Like the previous test, but the executor doesn't supply an image - it's pulled # from the origin on the run instead pods = DagsterKubernetesClient.production_client( ).core_api.list_namespaced_pod( namespace=helm_namespace_for_k8s_run_launcher) celery_pod_names = [ p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name ] check.invariant(not celery_pod_names) run_config = merge_dicts( load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env.yaml")), load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env_s3.yaml")), { "execution": { "k8s": { "config": { "job_namespace": helm_namespace_for_k8s_run_launcher, "image_pull_policy": image_pull_policy(), } } }, }, ) pipeline_name = "demo_k8s_executor_pipeline" run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher, run_config=run_config, pipeline_name=pipeline_name) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image( )
def test_k8s_run_launcher(dagster_instance, helm_namespace): run_config = load_yaml_from_path(os.path.join(test_project_environments_path(), "env.yaml")) pipeline_name = "demo_pipeline" run = create_run_for_test( dagster_instance, pipeline_name=pipeline_name, run_config=run_config, mode="default", ) dagster_instance.launch_run( run.run_id, ReOriginatedExternalPipelineForTest(get_test_project_external_pipeline(pipeline_name)), ) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace ) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_k8s_run_launcher_default( dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher ): # sanity check that we have a K8sRunLauncher check.inst(dagster_instance_for_k8s_run_launcher.run_launcher, K8sRunLauncher) pods = DagsterKubernetesClient.production_client().core_api.list_namespaced_pod( namespace=helm_namespace_for_k8s_run_launcher ) celery_pod_names = [p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name] check.invariant(not celery_pod_names) run_config = load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env.yaml")) pipeline_name = "demo_pipeline" tags = {"key": "value"} with get_test_project_external_pipeline_hierarchy( dagster_instance_for_k8s_run_launcher, pipeline_name ) as ( workspace, location, _repo, external_pipeline, ): reoriginated_pipeline = ReOriginatedExternalPipelineForTest(external_pipeline) run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=run_config, tags=tags, mode="default", pipeline_snapshot=external_pipeline.pipeline_snapshot, execution_plan_snapshot=location.get_external_execution_plan( external_pipeline, run_config, "default", None, None ).execution_plan_snapshot, external_pipeline_origin=reoriginated_pipeline.get_external_origin(), pipeline_code_origin=reoriginated_pipeline.get_python_origin(), ) dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher ) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result) updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image()
def test_execute_on_celery_k8s_with_resource_requirements( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace, dagit_url): run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) run_id = launch_run_over_graphql(dagit_url, run_config=run_config, pipeline_name="resources_limit_pipeline") result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_execute_on_celery_k8s_image_from_origin( # pylint: disable=redefined-outer-name dagster_docker_image, dagster_instance, helm_namespace): # Like the previous test, but the image is included in the pipeline origin # rather than in the executor config run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_engine_config(dagster_docker_image=None, job_namespace=helm_namespace), ) pipeline_name = "demo_pipeline_celery" with get_test_project_workspace_and_external_pipeline( dagster_instance, pipeline_name, container_image=dagster_docker_image) as (workspace, external_pipeline): reoriginated_pipeline = ReOriginatedExternalPipelineForTest( external_pipeline, container_image=dagster_docker_image) run = create_run_for_test( dagster_instance, pipeline_name=pipeline_name, run_config=run_config, mode="default", external_pipeline_origin=reoriginated_pipeline.get_external_origin( ), pipeline_code_origin=reoriginated_pipeline.get_python_origin(), ) dagster_instance.launch_run(run.run_id, workspace) result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format( result) updated_run = dagster_instance.get_run_by_id(run.run_id) assert updated_run.tags[DOCKER_IMAGE_TAG] == dagster_docker_image
def test_volume_mounts(dagster_docker_image, dagster_instance, helm_namespace, dagit_url): run_config = deep_merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env_s3.yaml") ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) run_id = launch_run_over_graphql( dagit_url, run_config=run_config, pipeline_name="volume_mount_pipeline", mode="celery", ) result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_failing_k8s_run_launcher(dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher): run_config = {"blah blah this is wrong": {}} pipeline_name = "demo_pipeline" with get_test_project_workspace_and_external_pipeline( dagster_instance_for_k8s_run_launcher, pipeline_name) as ( workspace, external_pipeline, ): reoriginated_pipeline = ReOriginatedExternalPipelineForTest( external_pipeline) run = create_run_for_test( dagster_instance_for_k8s_run_launcher, pipeline_name=pipeline_name, run_config=run_config, external_pipeline_origin=reoriginated_pipeline.get_external_origin( ), pipeline_code_origin=reoriginated_pipeline.get_python_origin(), ) dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format( result) event_records = dagster_instance_for_k8s_run_launcher.all_logs( run.run_id) assert any([ 'Received unexpected config entry "blah blah this is wrong"' in str(event) for event in event_records ]) assert any([ 'Missing required config entry "solids"' in str(event) for event in event_records ])
def test_k8s_run_launcher_secret_from_deployment( helm_namespace_for_k8s_run_launcher, dagit_url_for_k8s_run_launcher, ): # This run_config requires that WORD_FACTOR be set on both the user code deployment # and the run launcher. It will only work if secrets are propagated from the deployment # to the run launcher, since TEST_DEPLOYMENT_SECRET_NAME is only set on the user code # deployment but not on the run launcher config. run_config = load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env_config_from_secrets.yaml")) pipeline_name = "demo_pipeline" run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher, run_config=run_config, pipeline_name=pipeline_name) result = wait_for_job_and_get_raw_logs( job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_execute_subset_on_celery_k8s( # pylint: disable=redefined-outer-name dagster_docker_image, helm_namespace, dagit_url): run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env_subset.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), get_celery_engine_config(dagster_docker_image=dagster_docker_image, job_namespace=helm_namespace), ) run_id = launch_run_over_graphql( dagit_url, run_config=run_config, pipeline_name="demo_pipeline_celery", solid_selection=["count_letters"], ) result = wait_for_job_and_get_raw_logs(job_name="dagster-run-%s" % run_id, namespace=helm_namespace) assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def test_k8s_run_launcher_default(dagster_instance, helm_namespace): run_config = load_yaml_from_path( os.path.join(test_project_environments_path(), 'env.yaml')) pipeline_name = 'demo_pipeline' tags = {'key': 'value'} run = create_run_for_test( dagster_instance, pipeline_name=pipeline_name, run_config=run_config, tags=tags, mode='default', ) dagster_instance.launch_run( run.run_id, ReOriginatedExternalPipelineForTest( get_test_project_external_pipeline(pipeline_name)), ) result = wait_for_job_and_get_raw_logs(job_name='dagster-run-%s' % run.run_id, namespace=helm_namespace) assert 'PIPELINE_SUCCESS' in result, 'no match, result: {}'.format(result)