def test_terminate_launched_docker_run(): docker_image = get_test_project_docker_image() launcher_config = { "env_vars": ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY",], "network": "container:test-postgres-db-docker", } if IS_BUILDKITE: launcher_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_yamls([os.path.join(get_test_project_environments_path(), "env_s3.yaml"),]) with docker_postgres_instance( overrides={ "run_launcher": { "class": "DockerRunLauncher", "module": "dagster_docker", "config": launcher_config, } } ) as instance: recon_pipeline = get_test_project_recon_pipeline("hanging_pipeline", docker_image) run = instance.create_run_for_pipeline( pipeline_def=recon_pipeline.get_definition(), run_config=run_config, ) run_id = run.run_id external_pipeline = ReOriginatedExternalPipelineForTest( get_test_project_external_pipeline("hanging_pipeline", container_image=docker_image), container_image=docker_image, ) instance.launch_run(run_id, external_pipeline) poll_for_step_start(instance, run_id) assert instance.run_launcher.can_terminate(run_id) assert instance.run_launcher.terminate(run_id) terminated_pipeline_run = poll_for_finished_run(instance, run_id, timeout=30) terminated_pipeline_run = instance.get_run_by_id(run_id) assert terminated_pipeline_run.status == PipelineRunStatus.CANCELED run_logs = instance.all_logs(run_id) _check_event_log_contains( run_logs, [ ("PIPELINE_CANCELING", "Sending pipeline termination request"), ("STEP_FAILURE", 'Execution of step "hanging_solid" failed.'), ("PIPELINE_CANCELED", 'Execution of pipeline "hanging_pipeline" canceled.'), ("ENGINE_EVENT", "Pipeline execution terminated by interrupt"), ("ENGINE_EVENT", "Process for pipeline exited"), ], )
def test_launch_docker_image_on_pipeline_config(): # Docker image name to use for launch specified as part of the pipeline origin # rather than in the run launcher instance config docker_image = get_test_project_docker_image() launcher_config = { "env_vars": [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", ], "network": "container:test-postgres-db-docker", "container_kwargs": { "auto_remove": True, }, } if IS_BUILDKITE: launcher_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]) with docker_postgres_instance( overrides={ "run_launcher": { "class": "DockerRunLauncher", "module": "dagster_docker", "config": launcher_config, } }) as instance: recon_pipeline = get_test_project_recon_pipeline( "demo_pipeline", docker_image) run = instance.create_run_for_pipeline( pipeline_def=recon_pipeline.get_definition(), run_config=run_config, ) with get_test_project_external_pipeline( "demo_pipeline", container_image=docker_image) as orig_pipeline: external_pipeline = ReOriginatedExternalPipelineForTest( orig_pipeline, container_image=docker_image, ) instance.launch_run(run.run_id, external_pipeline) poll_for_finished_run(instance, run.run_id, timeout=60) run = instance.get_run_by_id(run.run_id) assert run.status == PipelineRunStatus.SUCCESS assert run.tags[DOCKER_IMAGE_TAG] == docker_image
def test_launch_docker_image_on_instance_config(): docker_image = get_test_project_docker_image() launcher_config = { "env_vars": [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", ], "network": "container:test-postgres-db-docker", "image": docker_image, } if IS_BUILDKITE: launcher_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]) with docker_postgres_instance( overrides={ "run_launcher": { "class": "DockerRunLauncher", "module": "dagster_docker", "config": launcher_config, } }) as instance: recon_pipeline = get_test_project_recon_pipeline("demo_pipeline") with get_test_project_workspace_and_external_pipeline( instance, "demo_pipeline") as ( workspace, orig_pipeline, ): external_pipeline = ReOriginatedExternalPipelineForTest( orig_pipeline) run = instance.create_run_for_pipeline( pipeline_def=recon_pipeline.get_definition(), run_config=run_config, external_pipeline_origin=external_pipeline.get_external_origin( ), pipeline_code_origin=external_pipeline.get_python_origin(), ) instance.launch_run(run.run_id, workspace) poll_for_finished_run(instance, run.run_id, timeout=60) assert instance.get_run_by_id( run.run_id).status == PipelineRunStatus.SUCCESS
def test_launch_docker_invalid_image(): docker_image = "_invalid_format_image" launcher_config = { "env_vars": [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", ], "network": "container:test-postgres-db-docker", "image": docker_image, } if IS_BUILDKITE: launcher_config["registry"] = get_buildkite_registry_config() run_config = merge_yamls( [ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ] ) with docker_postgres_instance( overrides={ "run_launcher": { "class": "DockerRunLauncher", "module": "dagster_docker", "config": launcher_config, } } ) as instance: recon_pipeline = get_test_project_recon_pipeline("demo_pipeline_s3") with get_test_project_workspace_and_external_pipeline(instance, "demo_pipeline_s3") as ( workspace, orig_pipeline, ): external_pipeline = ReOriginatedExternalPipelineForTest(orig_pipeline) run = instance.create_run_for_pipeline( pipeline_def=recon_pipeline.get_definition(), run_config=run_config, external_pipeline_origin=external_pipeline.get_external_origin(), pipeline_code_origin=external_pipeline.get_python_origin(), ) with pytest.raises( Exception, match=re.escape( "Docker image name _invalid_format_image is not correctly formatted" ), ): instance.launch_run(run.run_id, workspace)
def test_execute_celery_docker_image_on_pipeline_config(aws_creds): docker_image = get_test_project_docker_image() docker_config = { "network": "container:test-postgres-db-celery-docker", "container_kwargs": { "environment": [ "FIND_ME=here!", f"AWS_ACCESS_KEY_ID={aws_creds['aws_access_key_id']}", f"AWS_SECRET_ACCESS_KEY={aws_creds['aws_secret_access_key']}", ], # "auto_remove": False # uncomment when debugging to view container logs after execution }, } if IS_BUILDKITE: docker_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), os.path.join(get_test_project_environments_path(), "env_environment_vars.yaml"), ]), { "execution": { "celery-docker": { "config": { "docker": docker_config, "config_source": { "task_always_eager": True }, } } }, }, ) with celery_docker_postgres_instance() as instance: result = execute_pipeline( get_test_project_recon_pipeline("docker_celery_pipeline", docker_image), run_config=run_config, instance=instance, ) assert result.success assert result.result_for_solid("get_environment_solid").output_value( "result") == "here!"
def test_docker_executor(): """ Note that this test relies on having AWS credentials in the environment. """ executor_config = { "execution": { "docker": { "config": { "networks": ["container:test-postgres-db-docker"], "env_vars": [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", ], } } } } docker_image = get_test_project_docker_image() if IS_BUILDKITE: executor_config["execution"]["docker"]["config"][ "registry" ] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_dicts( merge_yamls( [ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ] ), executor_config, ) with environ({"DOCKER_LAUNCHER_NETWORK": "container:test-postgres-db-docker"}): with docker_postgres_instance() as instance: recon_pipeline = get_test_project_recon_pipeline("demo_pipeline_docker", docker_image) assert execute_pipeline( recon_pipeline, run_config=run_config, instance=instance ).success
def test_execute_celery_docker_image_on_pipeline_config(): docker_image = get_test_project_docker_image() docker_config = { "env_vars": [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", ], "network": "container:test-postgres-db-celery-docker", } if IS_BUILDKITE: docker_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), { "execution": { "celery-docker": { "config": { "docker": docker_config, "config_source": { "task_always_eager": True }, } } }, }, ) with celery_docker_postgres_instance() as instance: result = execute_pipeline( get_test_project_recon_pipeline("docker_celery_pipeline", docker_image), run_config=run_config, instance=instance, ) assert result.success
def _test_launch(docker_image, launcher_config, terminate=False): if IS_BUILDKITE: launcher_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_yamls( [ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ] ) with docker_postgres_instance( overrides={ "run_launcher": { "class": "DockerRunLauncher", "module": "dagster_docker", "config": launcher_config, } } ) as instance: recon_pipeline = get_test_project_recon_pipeline("demo_pipeline_s3") with get_test_project_workspace_and_external_pipeline(instance, "demo_pipeline_s3") as ( workspace, orig_pipeline, ): external_pipeline = ReOriginatedExternalPipelineForTest(orig_pipeline) run = instance.create_run_for_pipeline( pipeline_def=recon_pipeline.get_definition(), run_config=run_config, external_pipeline_origin=external_pipeline.get_external_origin(), pipeline_code_origin=external_pipeline.get_python_origin(), ) instance.launch_run(run.run_id, workspace) if not terminate: poll_for_finished_run(instance, run.run_id, timeout=60) assert instance.get_run_by_id(run.run_id).status == PipelineRunStatus.SUCCESS else: start_time = time.time() filters = PipelineRunsFilter( run_ids=[run.run_id], statuses=[ PipelineRunStatus.STARTED, ], ) while True: runs = instance.get_runs(filters, limit=1) if runs: break else: time.sleep(0.1) if time.time() - start_time > 60: raise Exception("Timed out waiting for run to start") launcher = instance.run_launcher assert launcher.can_terminate(run.run_id) assert launcher.terminate(run.run_id) poll_for_finished_run(instance, run.run_id, timeout=60) assert instance.get_run_by_id(run.run_id).status == PipelineRunStatus.CANCELED
def test_launch_docker_no_network(): docker_image = get_test_project_docker_image() launcher_config = { "env_vars": [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", ], } if IS_BUILDKITE: launcher_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_yamls( [ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ] ) with docker_postgres_instance( overrides={ "run_launcher": { "class": "DockerRunLauncher", "module": "dagster_docker", "config": launcher_config, } }, # Ensure the container will time out and fail quickly conn_args={ "params": {"connect_timeout": 2}, }, ) as instance: recon_pipeline = get_test_project_recon_pipeline("demo_pipeline_s3", docker_image) with get_test_project_workspace_and_external_pipeline( instance, "demo_pipeline_s3", container_image=docker_image ) as (workspace, orig_pipeline): external_pipeline = ReOriginatedExternalPipelineForTest( orig_pipeline, container_image=docker_image, ) run = instance.create_run_for_pipeline( pipeline_def=recon_pipeline.get_definition(), run_config=run_config, external_pipeline_origin=external_pipeline.get_external_origin(), pipeline_code_origin=external_pipeline.get_python_origin(), ) instance.launch_run(run.run_id, workspace) # Container launches, but run is stuck in STARTING state # due to not being able to access the network run = instance.get_run_by_id(run.run_id) assert run.tags[DOCKER_IMAGE_TAG] == docker_image container_id = run.tags[DOCKER_CONTAINER_ID_TAG] run = instance.get_run_by_id(run.run_id) assert run.status == PipelineRunStatus.STARTING assert run.tags[DOCKER_IMAGE_TAG] == docker_image client = docker.client.from_env() container = None try: start_time = time.time() while True: container = client.containers.get(container_id) if time.time() - start_time > 60: raise Exception("Timed out waiting for container to exit") if container.status == "exited": break time.sleep(3) finally: if container: container.remove(force=True)
def test_docker_monitoring(): docker_image = get_test_project_docker_image() launcher_config = { "env_vars": [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", ], "networks": ["container:test-postgres-db-docker"], "container_kwargs": { # "auto_remove": True, "volumes": ["/var/run/docker.sock:/var/run/docker.sock"], }, } if IS_BUILDKITE: launcher_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) run_config = merge_dicts( load_yaml_from_path( os.path.join(get_test_project_environments_path(), "env_s3.yaml")), { "solids": { "multiply_the_word_slow": { "inputs": { "word": "bar" }, "config": { "factor": 2, "sleep_time": 20 }, } }, "execution": { "docker": { "config": {} } }, }, ) with docker_postgres_instance({ "run_monitoring": { "enabled": True }, "run_launcher": { "class": "DockerRunLauncher", "module": "dagster_docker", "config": launcher_config, }, }) as instance: recon_pipeline = get_test_project_recon_pipeline( "demo_pipeline_docker_slow", docker_image) with get_test_project_workspace_and_external_pipeline( instance, "demo_pipeline_docker_slow", container_image=docker_image) as ( workspace, orig_pipeline, ): with start_daemon(): external_pipeline = ReOriginatedExternalPipelineForTest( orig_pipeline, container_image=docker_image) run = instance.create_run_for_pipeline( pipeline_def=recon_pipeline.get_definition(), run_config=run_config, external_pipeline_origin=external_pipeline. get_external_origin(), pipeline_code_origin=external_pipeline.get_python_origin(), ) with log_run_events(instance, run.run_id): instance.launch_run(run.run_id, workspace) start_time = time.time() while time.time() - start_time < 60: run = instance.get_run_by_id(run.run_id) if run.status == PipelineRunStatus.STARTED: break assert run.status == PipelineRunStatus.STARTING time.sleep(1) time.sleep(3) instance.run_launcher._get_container( # pylint:disable=protected-access instance.get_run_by_id(run.run_id)).stop() # daemon resumes the run poll_for_finished_run(instance, run.run_id, timeout=90) assert instance.get_run_by_id( run.run_id).status == PipelineRunStatus.SUCCESS
def test_container_context_on_pipeline(): docker_image = get_test_project_docker_image() launcher_config = {} if IS_BUILDKITE: launcher_config["registry"] = get_buildkite_registry_config() else: find_local_test_image(docker_image) executor_config = { "execution": { "docker": { "config": {} } }, } run_config = merge_dicts( merge_yamls([ os.path.join(get_test_project_environments_path(), "env.yaml"), os.path.join(get_test_project_environments_path(), "env_s3.yaml"), ]), executor_config, ) with docker_postgres_instance( overrides={ "run_launcher": { "class": "DockerRunLauncher", "module": "dagster_docker", "config": launcher_config, } }) as instance: recon_pipeline = get_test_project_recon_pipeline( "demo_pipeline_docker", docker_image, container_context={ "docker": { "env_vars": [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", ], "networks": ["container:test-postgres-db-docker"], "container_kwargs": { "auto_remove": True, "volumes": ["/var/run/docker.sock:/var/run/docker.sock"], }, } }, ) with get_test_project_workspace_and_external_pipeline( instance, "demo_pipeline_docker", container_image=docker_image) as ( workspace, orig_pipeline, ): external_pipeline = ReOriginatedExternalPipelineForTest( orig_pipeline, container_image=docker_image) run = instance.create_run_for_pipeline( pipeline_def=recon_pipeline.get_definition(), run_config=run_config, external_pipeline_origin=external_pipeline.get_external_origin( ), pipeline_code_origin=recon_pipeline.get_python_origin(), ) instance.launch_run(run.run_id, workspace) poll_for_finished_run(instance, run.run_id, timeout=60) for log in instance.all_logs(run.run_id): print(log) # pylint: disable=print-call assert instance.get_run_by_id( run.run_id).status == PipelineRunStatus.SUCCESS