def test_terminate_launched_docker_run():
docker_image = get_test_project_docker_image()
launcher_config = {
"env_vars": ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY",],
"network": "container:test-postgres-db-docker",
}
if IS_BUILDKITE:
launcher_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_yamls([os.path.join(get_test_project_environments_path(), "env_s3.yaml"),])
with docker_postgres_instance(
overrides={
"run_launcher": {
"class": "DockerRunLauncher",
"module": "dagster_docker",
"config": launcher_config,
}
}
) as instance:
recon_pipeline = get_test_project_recon_pipeline("hanging_pipeline", docker_image)
run = instance.create_run_for_pipeline(
pipeline_def=recon_pipeline.get_definition(), run_config=run_config,
)
run_id = run.run_id
external_pipeline = ReOriginatedExternalPipelineForTest(
get_test_project_external_pipeline("hanging_pipeline", container_image=docker_image),
container_image=docker_image,
)
instance.launch_run(run_id, external_pipeline)
poll_for_step_start(instance, run_id)
assert instance.run_launcher.can_terminate(run_id)
assert instance.run_launcher.terminate(run_id)
terminated_pipeline_run = poll_for_finished_run(instance, run_id, timeout=30)
terminated_pipeline_run = instance.get_run_by_id(run_id)
assert terminated_pipeline_run.status == PipelineRunStatus.CANCELED
run_logs = instance.all_logs(run_id)
_check_event_log_contains(
run_logs,
[
("PIPELINE_CANCELING", "Sending pipeline termination request"),
("STEP_FAILURE", 'Execution of step "hanging_solid" failed.'),
("PIPELINE_CANCELED", 'Execution of pipeline "hanging_pipeline" canceled.'),
("ENGINE_EVENT", "Pipeline execution terminated by interrupt"),
("ENGINE_EVENT", "Process for pipeline exited"),
],
)
def test_launch_docker_image_on_pipeline_config():
# Docker image name to use for launch specified as part of the pipeline origin
# rather than in the run launcher instance config
docker_image = get_test_project_docker_image()
launcher_config = {
"env_vars": [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
],
"network": "container:test-postgres-db-docker",
"container_kwargs": {
"auto_remove": True,
},
}
if IS_BUILDKITE:
launcher_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
])
with docker_postgres_instance(
overrides={
"run_launcher": {
"class": "DockerRunLauncher",
"module": "dagster_docker",
"config": launcher_config,
}
}) as instance:
recon_pipeline = get_test_project_recon_pipeline(
"demo_pipeline", docker_image)
run = instance.create_run_for_pipeline(
pipeline_def=recon_pipeline.get_definition(),
run_config=run_config,
)
with get_test_project_external_pipeline(
"demo_pipeline",
container_image=docker_image) as orig_pipeline:
external_pipeline = ReOriginatedExternalPipelineForTest(
orig_pipeline,
container_image=docker_image,
)
instance.launch_run(run.run_id, external_pipeline)
poll_for_finished_run(instance, run.run_id, timeout=60)
run = instance.get_run_by_id(run.run_id)
assert run.status == PipelineRunStatus.SUCCESS
assert run.tags[DOCKER_IMAGE_TAG] == docker_image
def test_launch_docker_image_on_instance_config():
docker_image = get_test_project_docker_image()
launcher_config = {
"env_vars": [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
],
"network": "container:test-postgres-db-docker",
"image": docker_image,
}
if IS_BUILDKITE:
launcher_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
])
with docker_postgres_instance(
overrides={
"run_launcher": {
"class": "DockerRunLauncher",
"module": "dagster_docker",
"config": launcher_config,
}
}) as instance:
recon_pipeline = get_test_project_recon_pipeline("demo_pipeline")
with get_test_project_workspace_and_external_pipeline(
instance, "demo_pipeline") as (
workspace,
orig_pipeline,
):
external_pipeline = ReOriginatedExternalPipelineForTest(
orig_pipeline)
run = instance.create_run_for_pipeline(
pipeline_def=recon_pipeline.get_definition(),
run_config=run_config,
external_pipeline_origin=external_pipeline.get_external_origin(
),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
instance.launch_run(run.run_id, workspace)
poll_for_finished_run(instance, run.run_id, timeout=60)
assert instance.get_run_by_id(
run.run_id).status == PipelineRunStatus.SUCCESS
def test_launch_docker_invalid_image():
docker_image = "_invalid_format_image"
launcher_config = {
"env_vars": [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
],
"network": "container:test-postgres-db-docker",
"image": docker_image,
}
if IS_BUILDKITE:
launcher_config["registry"] = get_buildkite_registry_config()
run_config = merge_yamls(
[
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]
)
with docker_postgres_instance(
overrides={
"run_launcher": {
"class": "DockerRunLauncher",
"module": "dagster_docker",
"config": launcher_config,
}
}
) as instance:
recon_pipeline = get_test_project_recon_pipeline("demo_pipeline_s3")
with get_test_project_workspace_and_external_pipeline(instance, "demo_pipeline_s3") as (
workspace,
orig_pipeline,
):
external_pipeline = ReOriginatedExternalPipelineForTest(orig_pipeline)
run = instance.create_run_for_pipeline(
pipeline_def=recon_pipeline.get_definition(),
run_config=run_config,
external_pipeline_origin=external_pipeline.get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
with pytest.raises(
Exception,
match=re.escape(
"Docker image name _invalid_format_image is not correctly formatted"
),
):
instance.launch_run(run.run_id, workspace)
def test_execute_celery_docker_image_on_pipeline_config(aws_creds):
docker_image = get_test_project_docker_image()
docker_config = {
"network": "container:test-postgres-db-celery-docker",
"container_kwargs": {
"environment": [
"FIND_ME=here!",
f"AWS_ACCESS_KEY_ID={aws_creds['aws_access_key_id']}",
f"AWS_SECRET_ACCESS_KEY={aws_creds['aws_secret_access_key']}",
],
# "auto_remove": False # uncomment when debugging to view container logs after execution
},
}
if IS_BUILDKITE:
docker_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
os.path.join(get_test_project_environments_path(),
"env_environment_vars.yaml"),
]),
{
"execution": {
"celery-docker": {
"config": {
"docker": docker_config,
"config_source": {
"task_always_eager": True
},
}
}
},
},
)
with celery_docker_postgres_instance() as instance:
result = execute_pipeline(
get_test_project_recon_pipeline("docker_celery_pipeline",
docker_image),
run_config=run_config,
instance=instance,
)
assert result.success
assert result.result_for_solid("get_environment_solid").output_value(
"result") == "here!"
def test_docker_executor():
"""
Note that this test relies on having AWS credentials in the environment.
"""
executor_config = {
"execution": {
"docker": {
"config": {
"networks": ["container:test-postgres-db-docker"],
"env_vars": [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
],
}
}
}
}
docker_image = get_test_project_docker_image()
if IS_BUILDKITE:
executor_config["execution"]["docker"]["config"][
"registry"
] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_dicts(
merge_yamls(
[
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]
),
executor_config,
)
with environ({"DOCKER_LAUNCHER_NETWORK": "container:test-postgres-db-docker"}):
with docker_postgres_instance() as instance:
recon_pipeline = get_test_project_recon_pipeline("demo_pipeline_docker", docker_image)
assert execute_pipeline(
recon_pipeline, run_config=run_config, instance=instance
).success
def test_execute_celery_docker_image_on_pipeline_config():
docker_image = get_test_project_docker_image()
docker_config = {
"env_vars": [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
],
"network": "container:test-postgres-db-celery-docker",
}
if IS_BUILDKITE:
docker_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
{
"execution": {
"celery-docker": {
"config": {
"docker": docker_config,
"config_source": {
"task_always_eager": True
},
}
}
},
},
)
with celery_docker_postgres_instance() as instance:
result = execute_pipeline(
get_test_project_recon_pipeline("docker_celery_pipeline",
docker_image),
run_config=run_config,
instance=instance,
)
assert result.success
def _test_launch(docker_image, launcher_config, terminate=False):
if IS_BUILDKITE:
launcher_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_yamls(
[
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]
)
with docker_postgres_instance(
overrides={
"run_launcher": {
"class": "DockerRunLauncher",
"module": "dagster_docker",
"config": launcher_config,
}
}
) as instance:
recon_pipeline = get_test_project_recon_pipeline("demo_pipeline_s3")
with get_test_project_workspace_and_external_pipeline(instance, "demo_pipeline_s3") as (
workspace,
orig_pipeline,
):
external_pipeline = ReOriginatedExternalPipelineForTest(orig_pipeline)
run = instance.create_run_for_pipeline(
pipeline_def=recon_pipeline.get_definition(),
run_config=run_config,
external_pipeline_origin=external_pipeline.get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
instance.launch_run(run.run_id, workspace)
if not terminate:
poll_for_finished_run(instance, run.run_id, timeout=60)
assert instance.get_run_by_id(run.run_id).status == PipelineRunStatus.SUCCESS
else:
start_time = time.time()
filters = PipelineRunsFilter(
run_ids=[run.run_id],
statuses=[
PipelineRunStatus.STARTED,
],
)
while True:
runs = instance.get_runs(filters, limit=1)
if runs:
break
else:
time.sleep(0.1)
if time.time() - start_time > 60:
raise Exception("Timed out waiting for run to start")
launcher = instance.run_launcher
assert launcher.can_terminate(run.run_id)
assert launcher.terminate(run.run_id)
poll_for_finished_run(instance, run.run_id, timeout=60)
assert instance.get_run_by_id(run.run_id).status == PipelineRunStatus.CANCELED
def test_launch_docker_no_network():
docker_image = get_test_project_docker_image()
launcher_config = {
"env_vars": [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
],
}
if IS_BUILDKITE:
launcher_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_yamls(
[
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]
)
with docker_postgres_instance(
overrides={
"run_launcher": {
"class": "DockerRunLauncher",
"module": "dagster_docker",
"config": launcher_config,
}
},
# Ensure the container will time out and fail quickly
conn_args={
"params": {"connect_timeout": 2},
},
) as instance:
recon_pipeline = get_test_project_recon_pipeline("demo_pipeline_s3", docker_image)
with get_test_project_workspace_and_external_pipeline(
instance, "demo_pipeline_s3", container_image=docker_image
) as (workspace, orig_pipeline):
external_pipeline = ReOriginatedExternalPipelineForTest(
orig_pipeline,
container_image=docker_image,
)
run = instance.create_run_for_pipeline(
pipeline_def=recon_pipeline.get_definition(),
run_config=run_config,
external_pipeline_origin=external_pipeline.get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
instance.launch_run(run.run_id, workspace)
# Container launches, but run is stuck in STARTING state
# due to not being able to access the network
run = instance.get_run_by_id(run.run_id)
assert run.tags[DOCKER_IMAGE_TAG] == docker_image
container_id = run.tags[DOCKER_CONTAINER_ID_TAG]
run = instance.get_run_by_id(run.run_id)
assert run.status == PipelineRunStatus.STARTING
assert run.tags[DOCKER_IMAGE_TAG] == docker_image
client = docker.client.from_env()
container = None
try:
start_time = time.time()
while True:
container = client.containers.get(container_id)
if time.time() - start_time > 60:
raise Exception("Timed out waiting for container to exit")
if container.status == "exited":
break
time.sleep(3)
finally:
if container:
container.remove(force=True)
def test_docker_monitoring():
docker_image = get_test_project_docker_image()
launcher_config = {
"env_vars": [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
],
"networks": ["container:test-postgres-db-docker"],
"container_kwargs": {
# "auto_remove": True,
"volumes": ["/var/run/docker.sock:/var/run/docker.sock"],
},
}
if IS_BUILDKITE:
launcher_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"solids": {
"multiply_the_word_slow": {
"inputs": {
"word": "bar"
},
"config": {
"factor": 2,
"sleep_time": 20
},
}
},
"execution": {
"docker": {
"config": {}
}
},
},
)
with docker_postgres_instance({
"run_monitoring": {
"enabled": True
},
"run_launcher": {
"class": "DockerRunLauncher",
"module": "dagster_docker",
"config": launcher_config,
},
}) as instance:
recon_pipeline = get_test_project_recon_pipeline(
"demo_pipeline_docker_slow", docker_image)
with get_test_project_workspace_and_external_pipeline(
instance,
"demo_pipeline_docker_slow",
container_image=docker_image) as (
workspace,
orig_pipeline,
):
with start_daemon():
external_pipeline = ReOriginatedExternalPipelineForTest(
orig_pipeline, container_image=docker_image)
run = instance.create_run_for_pipeline(
pipeline_def=recon_pipeline.get_definition(),
run_config=run_config,
external_pipeline_origin=external_pipeline.
get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
with log_run_events(instance, run.run_id):
instance.launch_run(run.run_id, workspace)
start_time = time.time()
while time.time() - start_time < 60:
run = instance.get_run_by_id(run.run_id)
if run.status == PipelineRunStatus.STARTED:
break
assert run.status == PipelineRunStatus.STARTING
time.sleep(1)
time.sleep(3)
instance.run_launcher._get_container( # pylint:disable=protected-access
instance.get_run_by_id(run.run_id)).stop()
# daemon resumes the run
poll_for_finished_run(instance, run.run_id, timeout=90)
assert instance.get_run_by_id(
run.run_id).status == PipelineRunStatus.SUCCESS
def test_container_context_on_pipeline():
docker_image = get_test_project_docker_image()
launcher_config = {}
if IS_BUILDKITE:
launcher_config["registry"] = get_buildkite_registry_config()
else:
find_local_test_image(docker_image)
executor_config = {
"execution": {
"docker": {
"config": {}
}
},
}
run_config = merge_dicts(
merge_yamls([
os.path.join(get_test_project_environments_path(), "env.yaml"),
os.path.join(get_test_project_environments_path(), "env_s3.yaml"),
]),
executor_config,
)
with docker_postgres_instance(
overrides={
"run_launcher": {
"class": "DockerRunLauncher",
"module": "dagster_docker",
"config": launcher_config,
}
}) as instance:
recon_pipeline = get_test_project_recon_pipeline(
"demo_pipeline_docker",
docker_image,
container_context={
"docker": {
"env_vars": [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
],
"networks": ["container:test-postgres-db-docker"],
"container_kwargs": {
"auto_remove": True,
"volumes":
["/var/run/docker.sock:/var/run/docker.sock"],
},
}
},
)
with get_test_project_workspace_and_external_pipeline(
instance, "demo_pipeline_docker",
container_image=docker_image) as (
workspace,
orig_pipeline,
):
external_pipeline = ReOriginatedExternalPipelineForTest(
orig_pipeline, container_image=docker_image)
run = instance.create_run_for_pipeline(
pipeline_def=recon_pipeline.get_definition(),
run_config=run_config,
external_pipeline_origin=external_pipeline.get_external_origin(
),
pipeline_code_origin=recon_pipeline.get_python_origin(),
)
instance.launch_run(run.run_id, workspace)
poll_for_finished_run(instance, run.run_id, timeout=60)
for log in instance.all_logs(run.run_id):
print(log) # pylint: disable=print-call
assert instance.get_run_by_id(
run.run_id).status == PipelineRunStatus.SUCCESS
