def test_engine_kill_SequentialTaskCollection():
with temporary_engine() as engine:
seq = SimpleSequentialTaskCollection(3)
engine.add(seq)
while seq.execution.state != 'RUNNING':
engine.progress()
# When the sequence is in RUNNING state, so must the first app
assert seq.tasks[0].execution.state == 'RUNNING'
assert seq.tasks[1].execution.state == 'NEW'
assert seq.tasks[2].execution.state == 'NEW'
# Killing a sequential should put all the applications in
# TERMINATED state. However, we will need an extra run of
# engine.progress() to update the status of all the jobs.
engine.kill(seq)
assert seq.tasks[0].execution.state == 'RUNNING'
assert seq.tasks[1].execution.state == 'NEW'
assert seq.tasks[2].execution.state == 'NEW'
assert seq.execution.state == 'RUNNING'
engine.progress()
assert seq.tasks[0].execution.state == 'TERMINATED'
assert seq.tasks[1].execution.state == 'TERMINATED'
assert seq.tasks[2].execution.state == 'TERMINATED'
assert seq.execution.state == 'TERMINATED'
def test_engine_forget_terminated(num_jobs=3,
transition_graph=None,
max_iter=100):
with temporary_engine() as engine:
engine.forget_terminated = True
# generate some no-op tasks
tasks = []
for n in range(num_jobs):
name = 'app{nr}'.format(nr=n + 1)
app = SuccessfulApp(name)
engine.add(app)
tasks.append(app)
# run them all
current_iter = 0
done = engine.counts()[Run.State.TERMINATED]
while done < num_jobs and current_iter < max_iter:
engine.progress()
done = engine.counts()[Run.State.TERMINATED]
current_iter += 1
# check that they have been forgotten
assert 0 == len(engine._managed.done)
for task in tasks:
assert task.execution.state == 'TERMINATED'
assert not task._attached
def test_engine_kill_ParallelTaskCollection():
# Creates an engine with 2 cores.
with temporary_engine(max_cores=2) as engine:
par = SimpleParallelTaskCollection(3)
engine.add(par)
for _ in range(20):
engine.progress()
if par.execution.state == 'RUNNING':
break
# Because of our noop engine, as soon as the parallel is in
# running we will have all jobs in SUBMITTED and the others in
# NEW.
assert (['SUBMITTED', 'SUBMITTED',
'NEW'] == [i.execution.state for i in par.tasks])
# Killing a parallel should put all the applications in
# TERMINATED state. However, we need two runs of
# engine.progress() to update the status of all the jobs
engine.kill(par)
assert (['SUBMITTED', 'SUBMITTED',
'NEW'] == [i.execution.state for i in par.tasks])
engine.progress()
engine.progress()
assert (['TERMINATED', 'TERMINATED',
'TERMINATED'] == [i.execution.state for i in par.tasks])
assert par.execution.state == 'TERMINATED'
def test_engine_kill_ParallelTaskCollection():
# Creates an engine with 2 cores.
with temporary_engine(max_cores=2) as engine:
par = SimpleParallelTaskCollection(3)
engine.add(par)
while par.execution.state != 'RUNNING':
engine.progress()
# Because of our noop engine, as soon as the parallel is in
# running we will have all jobs in SUBMITTED and the others in
# NEW.
assert_equal(
['TERMINATED', 'SUBMITTED', 'NEW'],
[i.execution.state for i in par.tasks],
)
# Killing a parallel should put all the applications in
# TERMINATED state. However, we need a run of
# engine.progress() to update the status of all the jobs
engine.kill(par)
assert_equal(
['TERMINATED', 'SUBMITTED', 'NEW'],
[i.execution.state for i in par.tasks],
)
engine.progress()
assert_equal(
['TERMINATED', 'TERMINATED', 'TERMINATED'],
[i.execution.state for i in par.tasks],
)
assert_equal(par.execution.state, 'TERMINATED')
def test_engine_kill_SequentialTaskCollection():
with temporary_engine() as engine:
seq = SimpleSequentialTaskCollection(3)
engine.add(seq)
while seq.execution.state != 'RUNNING':
engine.progress()
# Because of our noop engine, as soon as the sequential is in
# running we will have a job in TERMINATED and the others in
# NEW.
assert (
['TERMINATED', 'NEW', 'NEW'] ==
[i.execution.state for i in seq.tasks])
# Killing a sequential should put all the applications in
# TERMINATED state. However, we will need an extra run of
# engine.progress() to update the status of all the jobs.
engine.kill(seq)
assert (
['TERMINATED', 'NEW', 'NEW'] ==
[i.execution.state for i in seq.tasks])
engine.progress()
assert (
['TERMINATED', 'TERMINATED', 'TERMINATED'] ==
[i.execution.state for i in seq.tasks])
assert seq.execution.state == 'TERMINATED'
def test_engine_limits(limit_submitted,
limit_in_flight,
num_jobs=30,
max_iter=100):
"""
Test that `Engine.limit_in_flight` and `Engine.limit_submitted` are honored.
"""
with temporary_engine(max_cores=50) as engine:
# set limits
engine.max_in_flight = 10
engine.max_submitted = 2
# populate with test apps
apps = []
for n in range(num_jobs):
name = 'app{nr}'.format(nr=n)
app = SuccessfulApp(name)
engine.add(app)
apps.append(app)
# run all apps for (up to) a fixed nr of steps
iter_nr = 0
stats = engine.counts()
while stats['TERMINATED'] < num_jobs and iter_nr < max_iter:
iter_nr += 1
engine.progress()
stats = engine.counts()
submitted = stats['SUBMITTED']
assert submitted <= engine.max_submitted
in_flight = (stats['SUBMITTED'] + stats['RUNNING'])
assert in_flight <= engine.max_in_flight
# catch errors in case of termination because of exceeded iter count
assert stats["TERMINATED"] == num_jobs
def test_engine_kill_SequentialTaskCollection():
with temporary_engine() as engine:
seq = SimpleSequentialTaskCollection(3)
engine.add(seq)
while seq.execution.state != 'RUNNING':
engine.progress()
# Because of our noop engine, as soon as the sequential is in
# running we will have a job in TERMINATED and the others in
# NEW.
assert_equal(
['TERMINATED', 'NEW', 'NEW'],
[i.execution.state for i in seq.tasks],
)
# Killing a sequential should put all the applications in
# TERMINATED state. However, we will need an extra run of
# engine.progress() to update the status of all the jobs.
engine.kill(seq)
assert_equal(
['TERMINATED', 'NEW', 'NEW'],
[i.execution.state for i in seq.tasks],
)
engine.progress()
assert_equal(
['TERMINATED', 'TERMINATED', 'TERMINATED'],
[i.execution.state for i in seq.tasks],
)
assert_equal(seq.execution.state, 'TERMINATED')
def test_engine_progress_collection():
with temporary_engine() as engine:
seq = SimpleSequentialTaskCollection(3)
engine.add(seq)
# run through sequence
while seq.execution.state != 'TERMINATED':
engine.progress()
assert seq.stage().jobname == 'stage2'
assert seq.stage().execution.state == 'TERMINATED'
def test_engine_progress_collection():
with temporary_engine() as engine:
seq = SimpleSequentialTaskCollection(3)
engine.add(seq)
# run through sequence
while seq.execution.state != 'TERMINATED':
engine.progress()
assert seq.stage().jobname == 'stage2'
assert seq.stage().execution.state == 'TERMINATED'
def test_engine_redo_Task2():
"""Test that `Engine.redo()` raises if called on a Task that is not TERMINATED."""
with temporary_engine() as engine:
task = SuccessfulApp()
engine.add(task)
engine.progress()
assert task.execution.state != Run.State.NEW
# cannot redo a task that is not yet terminated
task.redo()
def test_engine_redo_ParallelTaskCollection():
with temporary_engine() as engine:
par = SimpleParallelTaskCollection(5)
engine.add(par)
# run until terminated
while par.execution.state != Run.State.TERMINATED:
engine.progress()
engine.redo(par)
assert par.execution.state == Run.State.NEW
for task in par.tasks:
assert task.execution.state == Run.State.NEW
def test_engine_redo_ParallelTaskCollection():
with temporary_engine() as engine:
par = SimpleParallelTaskCollection(5)
engine.add(par)
# run until terminated
while par.execution.state != Run.State.TERMINATED:
engine.progress()
engine.redo(par)
assert par.execution.state == Run.State.NEW
for task in par.tasks:
assert task.execution.state == Run.State.NEW
def test_engine_redo_Task2():
"""Test that `Engine.redo()` raises if called on a Task that is not TERMINATED."""
with temporary_engine() as engine:
task = SuccessfulApp()
engine.add(task)
engine.progress()
assert task.execution.state != Run.State.NEW
# cannot redo a task that is not yet terminated
with pytest.raises(AssertionError):
task.redo()
pytest.fail("`Task.redo()` succeeded on task not yet finished")
def test_engine_resources():
"""
Check that configured resources can be accessed through the `Engine` object.
"""
with temporary_engine() as engine:
resources = engine.resources
assert len(resources) == 1
assert 'test' in resources
test_rsc = resources['test']
# these should match the resource definition in `gc3libs.testing.helpers.temporary_core`
assert test_rsc.max_cores_per_job == 123
assert test_rsc.max_memory_per_core == 999 * GB
assert test_rsc.max_walltime == 7 * hours
def test_engine_submit1():
"""Engine.submit is equivalent to `add` if a task is not yet managed."""
with temporary_engine() as engine:
assert engine.counts()['NEW'] == 0
app = SuccessfulApp()
assert app.execution.state == 'NEW'
engine.submit(app)
assert app.execution.state == 'NEW'
assert engine.counts()['NEW'] == 1
engine.progress()
assert app.execution.state in ['SUBMITTED', 'RUNNING']
assert engine.counts()['NEW'] == 0
assert engine.counts()[app.execution.state] == 1
def test_engine_cannot_find_task_by_id_if_no_store():
"""
Test that `Engine.find_task_by_id` always raises `KeyError` if the Engine has no associated store.
"""
with temporary_engine() as engine:
with temporary_directory() as tmpdir:
store = FilesystemStore(tmpdir)
task = SuccessfulApp()
engine.add(task)
store.save(task) # guarantee it has a `.persistent_id`
task_id = task.persistent_id
with pytest.raises(KeyError):
engine.find_task_by_id(task_id)
def test_engine_submit1():
"""Engine.submit is equivalent to `add` if a task is not yet managed."""
with temporary_engine() as engine:
assert engine.stats()['NEW'] == 0
app = SuccessfulApp()
assert app.execution.state == 'NEW'
engine.submit(app)
assert app.execution.state == 'NEW'
assert engine.stats()['NEW'] == 1
engine.progress()
assert app.execution.state in ['SUBMITTED', 'RUNNING']
assert engine.stats()['NEW'] == 0
assert engine.stats()[app.execution.state] == 1
def test_engine_progress_collection_and_forget_terminated():
with temporary_engine() as engine:
engine.forget_terminated = True
seq = SimpleSequentialTaskCollection(3)
engine.add(seq)
# run through sequence
while seq.execution.state != 'TERMINATED':
engine.progress()
assert 0 == len(engine._managed.done)
assert not seq._attached
for task in seq.tasks:
assert not task._attached
def test_engine_progress(num_jobs=1, transition_graph=None, max_iter=100):
with temporary_engine() as engine:
# generate some no-op tasks
for n in range(num_jobs):
name = 'app{nr}'.format(nr=n+1)
engine.add(SuccessfulApp(name))
# run them all
current_iter = 0
done = engine.stats()[Run.State.TERMINATED]
while done < num_jobs and current_iter < max_iter:
engine.progress()
done = engine.stats()[Run.State.TERMINATED]
current_iter += 1
def test_engine_progress(num_jobs=1, transition_graph=None, max_iter=100):
with temporary_engine() as engine:
# generate some no-op tasks
for n in range(num_jobs):
name = 'app{nr}'.format(nr=n+1)
engine.add(SuccessfulApp(name))
# run them all
current_iter = 0
done = engine.stats()[Run.State.TERMINATED]
while done < num_jobs and current_iter < max_iter:
engine.progress()
done = engine.stats()[Run.State.TERMINATED]
current_iter += 1
def test_engine_resubmit():
with temporary_engine() as engine:
app = SuccessfulApp()
engine.add(app)
# run through sequence
while app.execution.state != 'TERMINATED':
engine.progress()
engine.submit(app, resubmit=True)
assert app.execution.state == 'NEW'
# run through sequence again
while app.execution.state != 'TERMINATED':
engine.progress()
assert app.execution.state == 'TERMINATED'
def test_engine_resubmit():
with temporary_engine() as engine:
app = SuccessfulApp()
engine.add(app)
# run through sequence
while app.execution.state != 'TERMINATED':
engine.progress()
engine.submit(app, resubmit=True)
assert app.execution.state == 'NEW'
# run through sequence again
while app.execution.state != 'TERMINATED':
engine.progress()
assert app.execution.state == 'TERMINATED'
def test_engine_redo_Task1():
"""Test correct use of `Engine.redo()` with a `Task` instance."""
with temporary_engine() as engine:
task = SuccessfulApp()
engine.add(task)
# run until terminated
while task.execution.state != Run.State.TERMINATED:
engine.progress()
assert task.execution.state == Run.State.TERMINATED
# no do it all over again
engine.redo(task)
assert task.execution.state == Run.State.NEW
engine.progress()
assert task.execution.state != Run.State.NEW
assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
def test_engine_redo_Task1():
"""Test correct use of `Engine.redo()` with a `Task` instance."""
with temporary_engine() as engine:
task = SuccessfulApp()
engine.add(task)
# run until terminated
while task.execution.state != Run.State.TERMINATED:
engine.progress()
assert task.execution.state == Run.State.TERMINATED
# no do it all over again
engine.redo(task)
assert task.execution.state == Run.State.NEW
engine.progress()
assert task.execution.state != Run.State.NEW
assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
def test_engine_submit2():
"""Engine.submit is a no-op if a task is already managed."""
with temporary_engine() as engine:
app = SuccessfulApp()
engine.add(app)
assert engine.stats()['NEW'] == 1
engine.submit(app)
assert engine.stats()['NEW'] == 1
engine.progress()
state = app.execution.state
assert state in ['SUBMITTED', 'RUNNING']
assert engine.stats()['NEW'] == 0
assert engine.stats()[state] == 1
engine.submit(app)
assert app.execution.state == state
assert engine.stats()[state] == 1
def test_engine_submit2():
"""Engine.submit is a no-op if a task is already managed."""
with temporary_engine() as engine:
app = SuccessfulApp()
engine.add(app)
assert engine.counts()['NEW'] == 1
engine.submit(app)
assert engine.counts()['NEW'] == 1
engine.progress()
state = app.execution.state
assert state in ['SUBMITTED', 'RUNNING']
assert engine.counts()['NEW'] == 0
assert engine.counts()[state] == 1
engine.submit(app)
assert app.execution.state == state
assert engine.counts()[state] == 1
def test_engine_progress(num_jobs=5, transition_graph=None, max_iter=100):
with temporary_engine() as engine:
# generate some no-op tasks
tasks = []
for n in range(num_jobs):
name = 'app{nr}'.format(nr=n + 1)
app = SuccessfulApp(name)
engine.add(app)
tasks.append(app)
# run them all
current_iter = 0
done = engine.counts()[Run.State.TERMINATED]
while done < num_jobs and current_iter < max_iter:
engine.progress()
done = engine.counts()[Run.State.TERMINATED]
current_iter += 1
# check state
for task in tasks:
assert task.execution.state == 'TERMINATED'
def _test_engine_counts(populate,
max_cores,
num_jobs,
num_new_jobs=None,
max_iter=100):
"""
Common code for the `test_engine_counts*` tests.
"""
# make the `.progress()` call a little less deterministic
transition_graph = {
Run.State.SUBMITTED: {
0.8: Run.State.RUNNING
},
Run.State.RUNNING: {
0.8: Run.State.TERMINATING
},
Run.State.TERMINATING: {
0.8: Run.State.TERMINATED
},
}
with temporary_engine(transition_graph, max_cores=max_cores) as engine:
# populate with test apps
apps = populate(engine)
# initial check on stats
actual_counts = engine.counts()
if num_new_jobs is None:
num_new_jobs = num_jobs
assert actual_counts['NEW'] == num_new_jobs
# check
iter_nr = 0
while (actual_counts['TERMINATED'] < num_jobs and iter_nr < max_iter):
iter_nr += 1
engine.progress()
actual_counts = engine.counts()
expected_counts = _compute_counts(apps)
_check_counts(actual_counts, expected_counts)
def test_engine_redo_Task3():
"""Test that `Engine.redo()` is a no-op if the Task is still NEW."""
with temporary_engine() as engine:
task = SuccessfulApp()
engine.add(task)
task.redo()
def test_engine_redo_Task3():
"""Test that `Engine.redo()` is a no-op if the Task is still NEW."""
with temporary_engine() as engine:
task = SuccessfulApp()
engine.add(task)
task.redo()