def _cli_api_execute_run_process(input_file, output_file, instance, pipeline_origin, pipeline_run):
write_unary_input(
input_file,
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
),
)
parts = [
pipeline_origin.executable_path,
'-m',
'dagster',
'api',
'execute_run',
input_file,
output_file,
]
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format(
pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id
),
pipeline_run,
engine_event_data=EngineEventData(marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def cli_api_execute_run_grpc(instance_ref, pipeline_origin, pipeline_run):
check.inst_param(instance_ref, 'instance_ref', InstanceRef)
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
instance = DagsterInstance.from_ref(instance_ref)
yield instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format(
pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id
),
pipeline_run,
engine_event_data=EngineEventData(marker_start='cli_api_subprocess_init'),
)
with ephemeral_grpc_api_client() as api_client:
execute_run_args = ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance_ref,
)
for event in api_client.execute_run(execute_run_args=execute_run_args):
if isinstance(event, IPCErrorMessage):
instance.report_engine_event(
event.message,
pipeline_run=pipeline_run,
engine_event_data=EngineEventData(
marker_end='cli_api_subprocess_init', error=event.serializable_error_info
),
)
instance.report_run_failed(pipeline_run)
return
yield event
def test_launch_run_grpc():
with instance_for_test() as instance:
with get_foo_grpc_pipeline_handle() as pipeline_handle:
api_client = pipeline_handle.repository_handle.repository_location_handle.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
res = api_client.start_run(
ExecuteRunArgs(
pipeline_origin=pipeline_handle.get_origin(),
pipeline_run_id=run_id,
instance_ref=instance.get_ref(),
))
assert res.success
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for pipeline exited")
event_records = instance.all_logs(run_id)
(started_process, executing_steps, finished_steps,
process_exited) = tuple(_get_engine_events(event_records))
assert "Started process for pipeline" in started_process.message
assert "Executing steps in process" in executing_steps.message
assert "Finished steps in process" in finished_steps.message
assert "Process for pipeline exited" in process_exited.message
def cli_api_execute_run(output_file, instance, pipeline_origin, pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
from dagster.cli.api import ExecuteRunArgsLoadComplete
with safe_tempfile_path() as input_file:
write_unary_input(
input_file,
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
),
)
parts = [
pipeline_origin.executable_path,
'-m',
'dagster',
'api',
'execute_run',
input_file,
output_file,
]
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
process = open_ipc_subprocess(parts)
# we need to process this event in order to ensure that the called process loads the input
event = next(ipc_read_event_stream(output_file))
check.inst(event, ExecuteRunArgsLoadComplete)
return process
def execute_run_grpc(api_client, instance_ref, pipeline_origin, pipeline_run):
'''Asynchronously execute a run over GRPC.'''
check.inst_param(api_client, 'api_client', DagsterGrpcClient)
check.inst_param(instance_ref, 'instance_ref', InstanceRef)
check.inst_param(pipeline_origin, 'pipeline_origin', PipelineOrigin)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
instance = DagsterInstance.from_ref(instance_ref)
yield instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
run_did_fail = False
execute_run_args = ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance_ref,
)
for event in api_client.execute_run(execute_run_args=execute_run_args):
if isinstance(event, IPCErrorMessage):
yield instance.report_engine_event(
event.message,
pipeline_run=pipeline_run,
engine_event_data=EngineEventData(
marker_end='cli_api_subprocess_init',
error=event.serializable_error_info),
)
if not run_did_fail:
run_did_fail = True
yield instance.report_run_failed(pipeline_run)
else:
yield event
def _cli_api_execute_run_process(input_file, output_file, instance,
pipeline_origin, pipeline_run):
write_unary_input(
input_file,
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
),
)
parts = [
pipeline_origin.executable_path,
"-m",
"dagster",
"api",
"execute_run",
input_file,
output_file,
]
return open_ipc_subprocess(parts)
def test_launch_unloadable_run_grpc():
with instance_for_test() as instance:
with get_foo_grpc_pipeline_handle() as pipeline_handle:
api_client = pipeline_handle.repository_handle.repository_location_handle.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
with instance_for_test() as other_instance:
res = api_client.start_run(
ExecuteRunArgs(
pipeline_origin=pipeline_handle.get_origin(),
pipeline_run_id=run_id,
instance_ref=other_instance.get_ref(),
))
assert not res.success
assert (
"gRPC server could not load run {run_id} in order to execute it. "
"Make sure that the gRPC server has access to your run storage."
.format(run_id=run_id)
in res.serializable_error_info.message)