def test_execution_plan_with_subset_snapshot_api_grpc():
with get_bar_repo_repository_location() as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
execution_plan_snapshot = sync_get_external_execution_plan_grpc(
api_client,
pipeline_handle.get_external_origin(),
run_config={
"solids": {
"do_input": {
"inputs": {
"x": {
"value": "test"
}
}
}
}
},
mode="default",
pipeline_snapshot_id="12345",
solid_selection=["do_input"],
)
assert isinstance(execution_plan_snapshot, ExecutionPlanSnapshot)
assert execution_plan_snapshot.step_keys_to_execute == [
"do_input",
]
assert len(execution_plan_snapshot.steps) == 1
def test_launch_run_grpc():
with instance_for_test() as instance:
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
res = deserialize_json_to_dagster_namedtuple(
api_client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=pipeline_handle.get_external_origin(),
pipeline_run_id=run_id,
instance_ref=instance.get_ref(),
)))
assert res.success
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for run exited")
event_records = instance.all_logs(run_id)
_check_event_log_contains(
event_records,
[("ENGINE_EVENT", msg) for msg in [
"Started process for run",
"Executing steps in process",
"Finished steps in process",
"Process for run exited",
]],
)
def test_pipeline_with_valid_subset_snapshot_api_grpc(instance):
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
external_pipeline_subset_result = _test_pipeline_subset_grpc(
pipeline_handle, api_client, ["do_something"])
assert isinstance(external_pipeline_subset_result,
ExternalPipelineSubsetResult)
assert external_pipeline_subset_result.success == True
assert external_pipeline_subset_result.external_pipeline_data.name == "foo"
def test_pipeline_with_invalid_subset_snapshot_api_grpc(instance):
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
with pytest.raises(
DagsterUserCodeProcessError,
match=
"No qualified solids to execute found for solid_selection",
):
_test_pipeline_subset_grpc(pipeline_handle, api_client,
["invalid_solid"])
def get_subset_external_pipeline_result(self, selector):
check.inst_param(selector, "selector", PipelineSelector)
check.invariant(
selector.location_name == self.name,
"PipelineSelector location_name mismatch, got {selector.location_name} expected {self.name}"
.format(self=self, selector=selector),
)
external_repository = self.get_repository(selector.repository_name)
pipeline_handle = PipelineHandle(selector.pipeline_name,
external_repository.handle)
return sync_get_external_pipeline_subset_grpc(
self.client, pipeline_handle.get_external_origin(),
selector.solid_selection)
def get_subset_external_pipeline_result(self, selector):
from dagster.api.snapshot_pipeline import sync_get_external_pipeline_subset
check.inst_param(selector, 'selector', PipelineSelector)
check.invariant(
selector.location_name == self.name,
'PipelineSelector location_name mismatch, got {selector.location_name} expected {self.name}'.format(
self=self, selector=selector
),
)
external_repository = self.external_repositories[selector.repository_name]
pipeline_handle = PipelineHandle(selector.pipeline_name, external_repository.handle)
return sync_get_external_pipeline_subset(pipeline_handle, selector.solid_selection)
def test_execution_plan_error_grpc(instance):
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo", repository_location.get_repository("bar_repo").handle
)
api_client = repository_location.client
with pytest.raises(
DagsterUserCodeProcessError,
match=re.escape("Could not find mode made_up_mode in pipeline foo"),
):
sync_get_external_execution_plan_grpc(
api_client,
pipeline_handle.get_external_origin(),
run_config={},
mode="made_up_mode",
pipeline_snapshot_id="12345",
)
def test_pipeline_with_invalid_definition_snapshot_api_grpc():
with get_bar_repo_handle() as repo_handle:
pipeline_handle = PipelineHandle("bar", repo_handle)
try:
_test_pipeline_subset_grpc(pipeline_handle, ["fail_subset"])
except DagsterUserCodeProcessError:
error_info = serializable_error_info_from_exc_info(sys.exc_info())
assert re.match(
(r".*DagsterInvalidSubsetError[\s\S]*"
r"The attempted subset \['fail_subset'\] for pipeline bar results in an invalid pipeline"
),
error_info.message,
)
assert re.match(
(r".*DagsterInvalidDefinitionError[\s\S]*"
r'add a dagster_type_loader for the type "InputTypeWithoutHydration"'
),
error_info.cause.message,
)
def test_pipeline_with_invalid_definition_snapshot_api():
pipeline_handle = PipelineHandle('bar', get_bar_repo_handle())
external_pipeline_subset_result = sync_get_external_pipeline_subset(
pipeline_handle.get_origin(), solid_selection=["fail_subset"])
assert isinstance(external_pipeline_subset_result,
ExternalPipelineSubsetResult)
assert external_pipeline_subset_result.success == False
assert re.match(
(r'.*DagsterInvalidSubsetError[\s\S]*'
r"The attempted subset \['fail_subset'\] for pipeline bar results in an invalid pipeline"
),
external_pipeline_subset_result.error.message,
)
assert re.match(
(r'.*DagsterInvalidDefinitionError[\s\S]*'
r'add a dagster_type_loader for the type "InputTypeWithoutHydration"'
),
external_pipeline_subset_result.error.cause.message,
)
def test_launch_unloadable_run_grpc():
with instance_for_test() as instance:
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
with instance_for_test() as other_instance:
res = deserialize_json_to_dagster_namedtuple(
api_client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=pipeline_handle.
get_external_origin(),
pipeline_run_id=run_id,
instance_ref=other_instance.get_ref(),
)))
assert not res.success
assert (
"gRPC server could not load run {run_id} in order to execute it. "
"Make sure that the gRPC server has access to your run storage."
.format(run_id=run_id)
in res.serializable_error_info.message)
def test_pipeline_with_invalid_definition_snapshot_api_grpc():
with get_bar_repo_handle() as repo_handle:
pipeline_handle = PipelineHandle("bar", repo_handle)
external_pipeline_subset_result = _test_pipeline_subset_grpc(
pipeline_handle, ["fail_subset"])
assert isinstance(external_pipeline_subset_result,
ExternalPipelineSubsetResult)
assert external_pipeline_subset_result.success == False
assert re.match(
(r".*DagsterInvalidSubsetError[\s\S]*"
r"The attempted subset \['fail_subset'\] for pipeline bar results in an invalid pipeline"
),
external_pipeline_subset_result.error.message,
)
assert re.match(
(r".*DagsterInvalidDefinitionError[\s\S]*"
r'add a dagster_type_loader for the type "InputTypeWithoutHydration"'
),
external_pipeline_subset_result.error.cause.message,
)
def test_execution_plan_with_step_keys_to_execute_snapshot_api_grpc(instance):
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo", repository_location.get_repository("bar_repo").handle
)
api_client = repository_location.client
execution_plan_snapshot = sync_get_external_execution_plan_grpc(
api_client,
pipeline_handle.get_external_origin(),
run_config={},
mode="default",
pipeline_snapshot_id="12345",
step_keys_to_execute=["do_something"],
)
assert isinstance(execution_plan_snapshot, ExecutionPlanSnapshot)
assert execution_plan_snapshot.step_keys_to_execute == [
"do_something",
]
assert len(execution_plan_snapshot.steps) == 2
def test_launch_run_with_unloadable_pipeline_grpc():
with instance_for_test() as instance:
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
original_origin = pipeline_handle.get_external_origin()
# point the api to a pipeline that cannot be loaded
res = deserialize_json_to_dagster_namedtuple(
api_client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=original_origin._replace(
pipeline_name="i_am_fake_pipeline"),
pipeline_run_id=run_id,
instance_ref=instance.get_ref(),
)))
assert res.success
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.FAILURE
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for run exited")
event_records = instance.all_logs(run_id)
_check_event_log_contains(
event_records,
[
("ENGINE_EVENT", "Started process for run"),
("ENGINE_EVENT", "Could not load pipeline definition"),
(
"PIPELINE_FAILURE",
"This run has been marked as failed from outside the execution context",
),
("ENGINE_EVENT", "Process for run exited"),
],
)
def legacy_get_foo_pipeline_handle():
return PipelineHandle('foo', legacy_get_bar_repo_handle())
def get_foo_grpc_pipeline_handle():
return PipelineHandle('foo', get_bar_grpc_repo_handle())
def get_foo_pipeline_handle():
return PipelineHandle("foo", get_bar_repo_handle())
def get_foo_grpc_pipeline_handle():
with get_bar_grpc_repo_handle() as repo_handle:
yield PipelineHandle("foo", repo_handle)