def test_has_run_query_and_terminate():
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'infinite_loop_pipeline')
instance = DagsterInstance.local_temp()
with safe_tempfile_path() as path:
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=infinite_loop_pipeline,
environment_dict={'solids': {
'loop': {
'config': {
'file': path
}
}
}},
)
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle, infinite_loop_pipeline,
pipeline_run, instance)
while not os.path.exists(path):
time.sleep(0.1)
assert os.path.exists(path)
assert execution_manager.is_process_running(pipeline_run.run_id)
assert execution_manager.terminate(pipeline_run.run_id)
assert instance.get_run_by_id(pipeline_run.run_id).is_finished
assert not execution_manager.is_process_running(pipeline_run.run_id)
assert not execution_manager.terminate(pipeline_run.run_id)
assert not os.path.exists(path)
def test_two_runs_running():
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'infinite_loop_pipeline')
with safe_tempfile_path() as file_one, safe_tempfile_path() as file_two:
instance = DagsterInstance.local_temp()
execution_manager = SubprocessExecutionManager(instance)
pipeline_run_one = instance.create_run_for_pipeline(
pipeline_def=infinite_loop_pipeline,
environment_dict={
'solids': {
'loop': {
'config': {
'file': file_one
}
}
}
},
)
execution_manager.execute_pipeline(handle, infinite_loop_pipeline,
pipeline_run_one, instance)
pipeline_run_two = instance.create_run_for_pipeline(
pipeline_def=infinite_loop_pipeline,
environment_dict={
'solids': {
'loop': {
'config': {
'file': file_two
}
}
}
},
)
execution_manager.execute_pipeline(handle, infinite_loop_pipeline,
pipeline_run_two, instance)
# ensure both runs have begun execution
while not os.path.exists(file_one) and not os.path.exists(file_two):
time.sleep(0.1)
assert execution_manager.is_process_running(pipeline_run_one.run_id)
assert execution_manager.is_process_running(pipeline_run_two.run_id)
assert execution_manager.terminate(pipeline_run_one.run_id)
assert not execution_manager.is_process_running(
pipeline_run_one.run_id)
assert execution_manager.is_process_running(pipeline_run_two.run_id)
assert execution_manager.terminate(pipeline_run_two.run_id)
assert not execution_manager.is_process_running(
pipeline_run_one.run_id)
assert not execution_manager.is_process_running(
pipeline_run_two.run_id)
def test_has_run_query_and_terminate():
run_id_one = make_new_run_id()
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'infinite_loop_pipeline')
instance = DagsterInstance.local_temp()
with get_temp_file_location() as path:
pipeline_run = instance.create_run(
PipelineRun.create_empty_run(
pipeline_name=infinite_loop_pipeline.name,
run_id=run_id_one,
environment_dict={
'solids': {
'loop': {
'config': {
'file': path
}
}
}
},
))
execution_manager = SubprocessExecutionManager(instance)
execution_manager.execute_pipeline(handle,
infinite_loop_pipeline,
pipeline_run,
instance,
raise_on_error=False)
while not os.path.exists(path):
time.sleep(0.1)
assert os.path.exists(path)
assert execution_manager.is_process_running(run_id_one)
assert execution_manager.terminate(run_id_one)
assert not execution_manager.is_process_running(run_id_one)
assert not execution_manager.terminate(run_id_one)
assert not os.path.exists(path)
def test_two_runs_running():
run_id_one = make_new_run_id()
run_id_two = make_new_run_id()
handle = ExecutionTargetHandle.for_pipeline_python_file(
__file__, 'infinite_loop_pipeline')
with get_temp_file_locations(2) as files:
file_one, file_two = files # pylint: disable=unbalanced-tuple-unpacking
instance = DagsterInstance.local_temp()
execution_manager = SubprocessExecutionManager(instance)
pipeline_run_one = instance.create_run(
PipelineRun.create_empty_run(
pipeline_name=infinite_loop_pipeline.name,
run_id=run_id_one,
environment_dict={
'solids': {
'loop': {
'config': {
'file': file_one
}
}
}
},
))
execution_manager.execute_pipeline(handle,
infinite_loop_pipeline,
pipeline_run_one,
instance,
raise_on_error=False)
pipeline_run_two = instance.create_run(
PipelineRun.create_empty_run(
pipeline_name=infinite_loop_pipeline.name,
run_id=run_id_two,
environment_dict={
'solids': {
'loop': {
'config': {
'file': file_two
}
}
}
},
))
execution_manager.execute_pipeline(handle,
infinite_loop_pipeline,
pipeline_run_two,
instance,
raise_on_error=False)
# ensure both runs have begun execution
while not os.path.exists(file_one) and not os.path.exists(file_two):
time.sleep(0.1)
assert execution_manager.is_process_running(run_id_one)
assert execution_manager.is_process_running(run_id_two)
assert execution_manager.terminate(run_id_one)
assert not execution_manager.is_process_running(run_id_one)
assert execution_manager.is_process_running(run_id_two)
assert execution_manager.terminate(run_id_two)
assert not execution_manager.is_process_running(run_id_one)
assert not execution_manager.is_process_running(run_id_two)