def launch_run(self, instance, run, external_pipeline=None):
'''Subclasses must implement this method.'''
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(run, 'run', PipelineRun)
check.inst_param(external_pipeline, 'external_pipeline',
ExternalPipeline)
env_handle = external_pipeline.handle.repository_handle.location_handle
# initialize when the first run happens
if not self._instance:
self.initialize(instance)
output_file = os.path.join(get_system_temp_directory(),
'cli-api-execute-run-{}'.format(run.run_id))
process = cli_api_execute_run(
output_file=output_file,
instance=self._instance,
repo_cli_args=env_handle.pointer.get_cli_args(),
pipeline_run=run,
)
with self._processes_lock:
self._living_process_by_run_id[run.run_id] = process
self._output_files_by_run_id[run.run_id] = output_file
return run
def test_execute_run_api(repo_handle):
with seven.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
pipeline_run = instance.create_run(
pipeline_name='foo',
run_id=None,
run_config={},
mode='default',
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
with safe_tempfile_path() as output_file_path:
process = cli_api_execute_run(
output_file=output_file_path,
instance=instance,
pipeline_origin=repo_handle.get_origin(),
pipeline_run=pipeline_run,
)
_stdout, _stderr = process.communicate()
events = [
event for event in ipc_read_event_stream(output_file_path)
]
assert len(events) == 12
assert [
event.event_type_value for event in events
if hasattr(event, 'event_type_value'
) # ExecuteRunArgsLoadComplete is synthetic
] == [
'PIPELINE_START',
'ENGINE_EVENT',
'STEP_START',
'STEP_OUTPUT',
'STEP_SUCCESS',
'STEP_START',
'STEP_INPUT',
'STEP_OUTPUT',
'STEP_SUCCESS',
'ENGINE_EVENT',
'PIPELINE_SUCCESS',
]
def execute_pipeline(
self, instance, external_pipeline, pipeline_run,
):
from dagster.api.execute_run import cli_api_execute_run
check.inst_param(instance, "instance", DagsterInstance)
check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
event_list = cli_api_execute_run(
instance=instance,
pipeline_origin=external_pipeline.get_origin(),
pipeline_run=pipeline_run,
)
return ExternalPipelineExecutionResult(event_list=event_list)
def test_execute_run_api(pipeline_handle):
with seven.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
events = cli_api_execute_run(
instance=instance,
pipeline_origin=pipeline_handle.get_origin(),
pipeline_run=pipeline_run,
)
assert len(events) == 14
assert [event.event_type_value for event in events] == [
"PIPELINE_START",
"ENGINE_EVENT",
"STEP_START",
"STEP_OUTPUT",
"OBJECT_STORE_OPERATION",
"STEP_SUCCESS",
"STEP_START",
"OBJECT_STORE_OPERATION",
"STEP_INPUT",
"STEP_OUTPUT",
"OBJECT_STORE_OPERATION",
"STEP_SUCCESS",
"ENGINE_EVENT",
"PIPELINE_SUCCESS",
]
def launch_run(self, instance, run, external_pipeline):
'''Subclasses must implement this method.'''
check.inst_param(run, 'run', PipelineRun)
check.inst_param(external_pipeline, 'external_pipeline',
ExternalPipeline)
output_file = os.path.join(get_system_temp_directory(),
'cli-api-execute-run-{}'.format(run.run_id))
process = cli_api_execute_run(
output_file=output_file,
instance=self._instance,
pipeline_origin=external_pipeline.get_origin(),
pipeline_run=run,
)
with self._processes_lock:
self._living_process_by_run_id[run.run_id] = process
self._output_files_by_run_id[run.run_id] = output_file
return run
def sync_cli_api_execute_run(instance, pipeline_origin, pipeline_name,
environment_dict, mode, solids_to_execute):
with safe_tempfile_path() as output_file_path:
pipeline_run = instance.create_run(
pipeline_name=pipeline_name,
run_id=None,
environment_dict=environment_dict,
mode=mode,
solids_to_execute=solids_to_execute,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
process = cli_api_execute_run(output_file_path, instance,
pipeline_origin, pipeline_run)
_stdout, _stderr = process.communicate()
for message in ipc_read_event_stream(output_file_path):
yield message