def test_max_concurrency_zero():
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'infinite_loop_pipeline')
with safe_tempfile_path() as filepath:
instance = DagsterInstance.local_temp()
execution_manager = QueueingSubprocessExecutionManager(
instance, max_concurrent_runs=0)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=infinite_loop_pipeline,
environment_dict={
'solids': {
'loop': {
'config': {
'file': filepath
}
}
}
},
)
execution_manager.execute_pipeline(handle, infinite_loop_pipeline,
pipeline_run, instance)
assert not execution_manager.is_active(pipeline_run.run_id)
assert not os.path.exists(filepath)
def test_max_concurrency_one():
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'infinite_loop_pipeline')
pipeline_def = handle.build_pipeline_definition()
with safe_tempfile_path() as file_one, safe_tempfile_path() as file_two:
instance = DagsterInstance.local_temp()
execution_manager = QueueingSubprocessExecutionManager(
instance, max_concurrent_runs=1)
run_one = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
environment_dict={
'solids': {
'loop': {
'config': {
'file': file_one
}
}
}
},
)
run_two = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
environment_dict={
'solids': {
'loop': {
'config': {
'file': file_two
}
}
}
},
)
execution_manager.execute_pipeline(handle, infinite_loop_pipeline,
run_one, instance)
execution_manager.execute_pipeline(handle, infinite_loop_pipeline,
run_two, instance)
while not os.path.exists(file_one):
execution_manager.check()
time.sleep(0.1)
assert execution_manager.is_active(run_one.run_id)
assert not execution_manager.is_active(run_two.run_id)
assert not os.path.exists(file_two)
assert execution_manager.terminate(run_one.run_id)
while not os.path.exists(file_two):
execution_manager.check()
time.sleep(0.1)
assert not execution_manager.is_active(run_one.run_id)
assert execution_manager.is_active(run_two.run_id)
assert execution_manager.terminate(run_two.run_id)