def test_running():
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'passing_pipeline')
environment_dict = {
'solids': {
'sum_solid': {
'inputs': {
'num': file_relative_path(__file__, 'data/num.csv')
}
}
}
}
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=passing_pipeline,
environment_dict=environment_dict,
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, passing_pipeline, pipeline_run,
instance)
execution_manager.join()
assert instance.get_run_by_id(
pipeline_run.run_id).status == PipelineRunStatus.SUCCESS
events = instance.all_logs(pipeline_run.run_id)
assert events
engine_events = get_events_of_type(events, DagsterEventType.ENGINE_EVENT)
assert (len([
ev for ev in engine_events
if 'SubprocessExecutionManager' in ev.message
]) == 3) # starting, started, exit
def test_execution_crash():
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'crashy_pipeline')
environment_dict = {
'solids': {
'sum_solid': {
'inputs': {
'num': file_relative_path(__file__, 'data/num.csv')
}
}
}
}
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=crashy_pipeline,
environment_dict=environment_dict,
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, crashy_pipeline, pipeline_run,
instance)
execution_manager.join()
assert instance.get_run_by_id(
pipeline_run.run_id).status == PipelineRunStatus.FAILURE
crash_log = instance.all_logs(pipeline_run.run_id)[
-2] # last message is pipeline failure, second to last is...
assert crash_log.message.startswith(
'[SubprocessExecutionManager] Pipeline execution process for {run_id} unexpectedly exited'
.format(run_id=pipeline_run.run_id))
def test_failing():
run_id = make_new_run_id()
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'failing_pipeline')
environment_dict = {
'solids': {
'sum_solid': {
'inputs': {
'num': file_relative_path(__file__, 'data/num.csv')
}
}
}
}
selector = ExecutionSelector('csv_hello_world')
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run(
PipelineRun(
pipeline_name=failing_pipeline.name,
run_id=run_id,
selector=selector,
environment_dict=environment_dict,
mode='default',
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.NOT_STARTED,
))
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, failing_pipeline, pipeline_run,
instance)
execution_manager.join()
assert instance.get_run_by_id(run_id).status == PipelineRunStatus.FAILURE
assert instance.all_logs(run_id)
def test_failing():
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'failing_pipeline')
environment_dict = {
'solids': {
'sum_solid': {
'inputs': {
'num': file_relative_path(__file__, 'data/num.csv')
}
}
}
}
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=failing_pipeline,
environment_dict=environment_dict,
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, failing_pipeline, pipeline_run,
instance)
execution_manager.join()
assert instance.get_run_by_id(
pipeline_run.run_id).status == PipelineRunStatus.FAILURE
assert instance.all_logs(pipeline_run.run_id)
def test_execution_crash():
run_id = make_new_run_id()
handle = ExecutionTargetHandle.for_pipeline_python_file(__file__, 'crashy_pipeline')
environment_dict = {
'solids': {'sum_solid': {'inputs': {'num': file_relative_path(__file__, 'data/num.csv')}}}
}
selector = ExecutionSelector('csv_hello_world')
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run(
PipelineRun(
pipeline_name=crashy_pipeline.name,
run_id=run_id,
selector=selector,
environment_dict=environment_dict,
mode='default',
reexecution_config=None,
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.NOT_STARTED,
)
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, crashy_pipeline, pipeline_run, instance)
execution_manager.join()
assert instance.get_run_by_id(run_id).status == PipelineRunStatus.FAILURE
last_log = instance.all_logs(run_id)[-1]
assert last_log.message.startswith(
'Exception: Pipeline execution process for {run_id} unexpectedly exited\n'.format(
run_id=run_id
)
)
def test_multiprocessing_execution_for_composite_solid():
environment_dict = {
'solids': {
'composite_with_nested_config_solid': {
'solids': {'node_a': {'config': {'foo': 'baz'}}, 'node_b': {'config': {'bar': 3}}}
}
}
}
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=composite_pipeline, environment_dict=environment_dict,
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(reconstructable(composite_pipeline), pipeline_run, instance)
execution_manager.join()
assert instance.get_run_by_id(pipeline_run.run_id).status == PipelineRunStatus.SUCCESS
environment_dict = {
'solids': {
'composite_with_nested_config_solid': {
'solids': {'node_a': {'config': {'foo': 'baz'}}, 'node_b': {'config': {'bar': 3}}}
}
},
'execution': {'multiprocess': {}},
'storage': {'filesystem': {}},
}
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=composite_pipeline, environment_dict=environment_dict,
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(reconstructable(composite_pipeline), pipeline_run, instance)
execution_manager.join()
def test_multiprocessing_execution_for_composite_solid():
environment_dict = {
'solids': {
'composite_with_nested_config_solid': {
'solids': {'node_a': {'config': {'foo': 'baz'}}, 'node_b': {'config': {'bar': 3}}}
}
}
}
run_id = make_new_run_id()
handle = ExecutionTargetHandle.for_pipeline_python_file(__file__, 'composite_pipeline')
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run(
PipelineRun(
pipeline_name=composite_pipeline.name,
run_id=run_id,
selector=ExecutionSelector('nonce'),
environment_dict=environment_dict,
mode='default',
reexecution_config=None,
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.NOT_STARTED,
)
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, composite_pipeline, pipeline_run, instance)
execution_manager.join()
assert instance.get_run_by_id(run_id).status == PipelineRunStatus.SUCCESS
environment_dict = {
'solids': {
'composite_with_nested_config_solid': {
'solids': {'node_a': {'config': {'foo': 'baz'}}, 'node_b': {'config': {'bar': 3}}}
}
},
'execution': {'multiprocess': {}},
'storage': {'filesystem': {}},
}
run_id = make_new_run_id()
pipeline_run = instance.create_run(
PipelineRun(
pipeline_name=composite_pipeline.name,
run_id=run_id,
selector=ExecutionSelector('nonce'),
environment_dict=environment_dict,
mode='default',
reexecution_config=None,
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.NOT_STARTED,
)
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, composite_pipeline, pipeline_run, instance)
execution_manager.join()
def test_multiprocessing_execution_for_composite_solid_with_config_mapping():
environment_dict = {
'solids': {
'composite_with_nested_config_solid_and_config_mapping': {
'config': {
'foo': 'baz',
'bar': 3
}
}
}
}
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'composite_pipeline_with_config_mapping')
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=composite_pipeline_with_config_mapping,
environment_dict=environment_dict,
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle,
composite_pipeline_with_config_mapping,
pipeline_run, instance)
execution_manager.join()
assert instance.get_run_by_id(
pipeline_run.run_id).status == PipelineRunStatus.SUCCESS
environment_dict = {
'solids': {
'composite_with_nested_config_solid_and_config_mapping': {
'config': {
'foo': 'baz',
'bar': 3
}
}
},
'execution': {
'multiprocess': {}
},
'storage': {
'filesystem': {}
},
}
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=composite_pipeline_with_config_mapping,
environment_dict=environment_dict,
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle,
composite_pipeline_with_config_mapping,
pipeline_run, instance)
execution_manager.join()
assert instance.get_run_by_id(
pipeline_run.run_id).status == PipelineRunStatus.SUCCESS
def test_running():
run_id = make_new_run_id()
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'passing_pipeline')
environment_dict = {
'solids': {
'sum_solid': {
'inputs': {
'num': script_relative_path('data/num.csv')
}
}
}
}
selector = ExecutionSelector('csv_hello_world')
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run(
PipelineRun(
pipeline_name=passing_pipeline.name,
run_id=run_id,
selector=selector,
environment_dict=environment_dict,
mode='default',
reexecution_config=None,
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.NOT_STARTED,
))
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle,
passing_pipeline,
pipeline_run,
instance,
raise_on_error=False)
execution_manager.join()
assert instance.get_run(run_id).status == PipelineRunStatus.SUCCESS
events = instance.all_logs(run_id)
assert events
process_start_events = get_events_of_type(
events, DagsterEventType.PIPELINE_PROCESS_START)
assert len(process_start_events) == 1
process_started_events = get_events_of_type(
events, DagsterEventType.PIPELINE_PROCESS_STARTED)
assert len(process_started_events) == 1
process_exited_events = get_events_of_type(
events, DagsterEventType.PIPELINE_PROCESS_EXITED)
assert len(process_exited_events) == 1
def test_running():
run_id = make_new_run_id()
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'passing_pipeline')
environment_dict = {
'solids': {
'sum_solid': {
'inputs': {
'num': file_relative_path(__file__, 'data/num.csv')
}
}
}
}
selector = ExecutionSelector('csv_hello_world')
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run(
PipelineRun(
pipeline_name=passing_pipeline.name,
run_id=run_id,
selector=selector,
environment_dict=environment_dict,
mode='default',
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.NOT_STARTED,
))
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, passing_pipeline, pipeline_run,
instance)
execution_manager.join()
assert instance.get_run_by_id(run_id).status == PipelineRunStatus.SUCCESS
events = instance.all_logs(run_id)
assert events
engine_events = get_events_of_type(events, DagsterEventType.ENGINE_EVENT)
assert (len([
ev for ev in engine_events
if 'SubprocessExecutionManager' in ev.message
]) == 3) # starting, started, exit
