def test_execution_plan_with_subset_snapshot_api_grpc():
with get_bar_repo_repository_location() as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
execution_plan_snapshot = sync_get_external_execution_plan_grpc(
api_client,
pipeline_handle.get_external_origin(),
run_config={
"solids": {
"do_input": {
"inputs": {
"x": {
"value": "test"
}
}
}
}
},
mode="default",
pipeline_snapshot_id="12345",
solid_selection=["do_input"],
)
assert isinstance(execution_plan_snapshot, ExecutionPlanSnapshot)
assert execution_plan_snapshot.step_keys_to_execute == [
"do_input",
]
assert len(execution_plan_snapshot.steps) == 1
def test_launch_run_grpc():
with instance_for_test() as instance:
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
res = deserialize_json_to_dagster_namedtuple(
api_client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=pipeline_handle.get_external_origin(),
pipeline_run_id=run_id,
instance_ref=instance.get_ref(),
)))
assert res.success
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for run exited")
event_records = instance.all_logs(run_id)
_check_event_log_contains(
event_records,
[("ENGINE_EVENT", msg) for msg in [
"Started process for run",
"Executing steps in process",
"Finished steps in process",
"Process for run exited",
]],
)
def get_subset_external_pipeline_result(self, selector):
check.inst_param(selector, "selector", PipelineSelector)
check.invariant(
selector.location_name == self.name,
"PipelineSelector location_name mismatch, got {selector.location_name} expected {self.name}"
.format(self=self, selector=selector),
)
external_repository = self.get_repository(selector.repository_name)
pipeline_handle = PipelineHandle(selector.pipeline_name,
external_repository.handle)
return sync_get_external_pipeline_subset_grpc(
self.client, pipeline_handle.get_external_origin(),
selector.solid_selection)
def test_execution_plan_error_grpc(instance):
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo", repository_location.get_repository("bar_repo").handle
)
api_client = repository_location.client
with pytest.raises(
DagsterUserCodeProcessError,
match=re.escape("Could not find mode made_up_mode in pipeline foo"),
):
sync_get_external_execution_plan_grpc(
api_client,
pipeline_handle.get_external_origin(),
run_config={},
mode="made_up_mode",
pipeline_snapshot_id="12345",
)
def test_execution_plan_with_step_keys_to_execute_snapshot_api_grpc(instance):
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo", repository_location.get_repository("bar_repo").handle
)
api_client = repository_location.client
execution_plan_snapshot = sync_get_external_execution_plan_grpc(
api_client,
pipeline_handle.get_external_origin(),
run_config={},
mode="default",
pipeline_snapshot_id="12345",
step_keys_to_execute=["do_something"],
)
assert isinstance(execution_plan_snapshot, ExecutionPlanSnapshot)
assert execution_plan_snapshot.step_keys_to_execute == [
"do_something",
]
assert len(execution_plan_snapshot.steps) == 2
def test_launch_run_with_unloadable_pipeline_grpc():
with instance_for_test() as instance:
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
original_origin = pipeline_handle.get_external_origin()
# point the api to a pipeline that cannot be loaded
res = deserialize_json_to_dagster_namedtuple(
api_client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=original_origin._replace(
pipeline_name="i_am_fake_pipeline"),
pipeline_run_id=run_id,
instance_ref=instance.get_ref(),
)))
assert res.success
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.FAILURE
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for run exited")
event_records = instance.all_logs(run_id)
_check_event_log_contains(
event_records,
[
("ENGINE_EVENT", "Started process for run"),
("ENGINE_EVENT", "Could not load pipeline definition"),
(
"PIPELINE_FAILURE",
"This run has been marked as failed from outside the execution context",
),
("ENGINE_EVENT", "Process for run exited"),
],
)
