def test_staged_task_collection_progress():
class ThreeStageCollection(StagedTaskCollection):
def stage0(self):
return SuccessfulApp()
def stage1(self):
return SuccessfulApp()
def stage2(self):
return UnsuccessfulApp()
with temporary_core() as core:
coll = ThreeStageCollection()
coll.attach(core)
coll.submit()
assert coll.execution.state == Run.State.SUBMITTED
# first task is successful
while coll.tasks[0].execution.state != Run.State.TERMINATED:
coll.progress()
assert coll.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
#assert_equal(coll.execution.exitcode, 0)
# second task is successful
while coll.tasks[1].execution.state != Run.State.TERMINATED:
coll.progress()
#assert_equal(coll.execution.state, Run.State.RUNNING)
#assert_equal(coll.execution.exitcode, 0)
# third task is unsuccessful
while coll.tasks[2].execution.state != Run.State.TERMINATED:
coll.progress()
assert coll.execution.state == Run.State.TERMINATED
assert coll.execution.exitcode == 1
def test_task_state_handlers():
report = {
'submitted': False,
'terminating': False,
'terminated': False,
}
class HandlerTestApp(SuccessfulApp):
def __init__(self, report):
super(HandlerTestApp, self).__init__()
self.report = report
def submitted(self):
self.report['submitted'] = True
print("Submitted!")
def terminating(self):
self.report['terminating'] = True
print("Terminating!")
def terminated(self):
print("Terminated!")
self.report['terminated'] = True
with temporary_core() as core:
task = HandlerTestApp(report)
task.attach(core)
task.submit()
assert report['submitted']
# run until terminated
while task.execution.state != Run.State.TERMINATED:
task.progress()
assert report['terminating']
assert report['terminated']
def test_staged_task_collection_progress():
class ThreeStageCollection(StagedTaskCollection):
def stage0(self):
return SuccessfulApp()
def stage1(self):
return SuccessfulApp()
def stage2(self):
return UnsuccessfulApp()
with temporary_core() as core:
coll = ThreeStageCollection()
coll.attach(core)
coll.submit()
assert coll.execution.state == Run.State.SUBMITTED
# first task is successful
while coll.tasks[0].execution.state != Run.State.TERMINATED:
coll.progress()
assert coll.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
#assert_equal(coll.execution.exitcode, 0)
# second task is successful
while coll.tasks[1].execution.state != Run.State.TERMINATED:
coll.progress()
#assert_equal(coll.execution.state, Run.State.RUNNING)
#assert_equal(coll.execution.exitcode, 0)
# third task is unsuccessful
while coll.tasks[2].execution.state != Run.State.TERMINATED:
coll.progress()
assert coll.execution.state == Run.State.TERMINATED
assert coll.execution.exitcode == 1
def test_shellcmd_backend_create_spooldir():
with temporary_directory() as tmpdir:
with temporary_core(spooldir=tmpdir) as core:
assert 'test' in core.resources
backend = core.get_backend('test')
app = SuccessfulApp()
core.submit(app)
assert os.path.isdir(backend.spooldir)
def test_ParallelTaskCollection_progress():
with temporary_core(max_cores=10) as core:
par = SimpleParallelTaskCollection(5)
par.attach(core)
# run until terminated
while par.execution.state != Run.State.TERMINATED:
par.progress()
for task in par.tasks:
assert task.execution.state == Run.State.TERMINATED
def test_SequentialTaskCollection_progress():
with temporary_core() as core:
seq = SimpleSequentialTaskCollection(3)
seq.attach(core)
# run until terminated
while seq.execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.stage().jobname == 'stage2'
assert seq.stage().execution.state == Run.State.TERMINATED
def test_empty_SequentialTaskCollection_progress():
with temporary_core() as core:
seq = SimpleSequentialTaskCollection(0)
seq.attach(core)
# run until terminated
while seq.execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.execution.state == Run.State.TERMINATED
assert seq.execution.returncode == 0
def test_ParallelTaskCollection_progress():
with temporary_core(max_cores=10) as core:
par = SimpleParallelTaskCollection(5)
par.attach(core)
# run until terminated
while par.execution.state != Run.State.TERMINATED:
par.progress()
for task in par.tasks:
assert task.execution.state == Run.State.TERMINATED
def test_SequentialTaskCollection_progress():
with temporary_core() as core:
seq = SimpleSequentialTaskCollection(3)
seq.attach(core)
# run until terminated
while seq.execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.stage().jobname == 'stage2'
assert seq.stage().execution.state == Run.State.TERMINATED
def test_task_redo2():
"""Test that `.redo()` raises if called on a Task that is not TERMINATED."""
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.submit()
assert task.execution.state == Run.State.SUBMITTED
# cannot redo a task that is not yet terminated
task.redo()
def test_task_progress():
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.submit()
assert task.execution.state == Run.State.SUBMITTED
# run until terminated
while task.execution.state != Run.State.TERMINATED:
task.progress()
assert task.execution.state == Run.State.TERMINATED
def test_task_progress():
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.submit()
assert task.execution.state == Run.State.SUBMITTED
# run until terminated
while task.execution.state != Run.State.TERMINATED:
task.progress()
assert task.execution.state == Run.State.TERMINATED
def test_task_redo2():
"""Test that `.redo()` raises if called on a Task that is not TERMINATED."""
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.submit()
assert task.execution.state == Run.State.SUBMITTED
# cannot redo a task that is not yet terminated
with pytest.raises(AssertionError,
message="`Task.redo()` succeeded on task not yet finished"):
task.redo()
def test_task_redo2():
"""Test that `.redo()` raises if called on a Task that is not TERMINATED."""
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.submit()
assert task.execution.state == Run.State.SUBMITTED
# cannot redo a task that is not yet terminated
with pytest.raises(AssertionError):
task.redo()
pytest.fail("`Task.redo()` succeeded on task not yet finished")
def test_core_resources():
"""
Check that configured resources can be accessed through the `Core` object.
"""
with temporary_core() as core:
resources = core.resources
assert len(resources) == 1
assert 'test' in resources
test_rsc = resources['test']
# these should match the resource definition in `gc3libs.testing.helpers.temporary_core`
assert test_rsc.max_cores_per_job == 1
assert test_rsc.max_memory_per_core == 1 * GB
assert test_rsc.max_walltime == 8 * hours
def test_engine_find_task_by_id():
"""
Test that saved tasks are can be retrieved from the Engine given their ID only.
"""
with temporary_core() as core:
with temporary_directory() as tmpdir:
store = FilesystemStore(tmpdir)
engine = Engine(core, store=store)
task = SuccessfulApp()
store.save(task)
engine.add(task)
task_id = task.persistent_id
assert engine.find_task_by_id(task_id) == task
def test_engine_cannot_find_task_by_id_if_not_saved():
"""
Test that *unsaved* tasks are cannot be retrieved from the Engine given their ID only.
"""
with temporary_core() as core:
with temporary_directory() as tmpdir:
store = FilesystemStore(tmpdir)
engine = Engine(core, store=store)
task = SuccessfulApp()
engine.add(task)
store.save(task) # guarantee it has a `.persistent_id`
task_id = task.persistent_id
with pytest.raises(KeyError):
engine.find_task_by_id(task_id)
def test_task_redo1():
"""Test correct use of `Task.redo()`"""
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.submit()
assert task.execution.state == Run.State.SUBMITTED
# run until terminated
while task.execution.state != Run.State.TERMINATED:
task.progress()
assert task.execution.state == Run.State.TERMINATED
# no do it all over again
task.redo()
assert task.execution.state == Run.State.NEW
task.progress()
assert task.execution.state != Run.State.NEW
assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
def test_task_redo1():
"""Test correct use of `Task.redo()`"""
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.submit()
assert task.execution.state == Run.State.SUBMITTED
# run until terminated
while task.execution.state != Run.State.TERMINATED:
task.progress()
assert task.execution.state == Run.State.TERMINATED
# no do it all over again
task.redo()
assert task.execution.state == Run.State.NEW
task.progress()
assert task.execution.state != Run.State.NEW
assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
def test_staged_task_collection_stage():
class TwoStageCollection(StagedTaskCollection):
def stage0(self):
return SuccessfulApp(name='stage0')
def stage1(self):
return UnsuccessfulApp(name='stage1')
with temporary_core() as core:
coll = TwoStageCollection()
coll.attach(core)
coll.submit()
stage = coll.stage()
assert isinstance(stage, SuccessfulApp), ("stage=%r" % (stage,))
assert stage.jobname == 'stage0'
# advance to next task
while coll.tasks[0].execution.state != Run.State.TERMINATED:
coll.progress()
stage = coll.stage()
assert isinstance(stage, UnsuccessfulApp)
assert stage.jobname == 'stage1'
def test_SequentialTaskCollection_redo2():
with temporary_core() as core:
seq = SimpleSequentialTaskCollection(3)
seq.attach(core)
# run until terminated
while seq.execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.stage().jobname == "stage2"
assert seq.stage().execution.state == Run.State.TERMINATED
seq.redo(1)
assert seq.stage().jobname == "stage1"
assert seq.stage().execution.state == Run.State.NEW
assert seq.execution.state == Run.State.NEW
# run until terminated, again
while seq.execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.stage().jobname == "stage2"
assert seq.stage().execution.state == Run.State.TERMINATED
def test_shellcmd_backend_parse_wrapper_output_successful():
resource_usage = StringIO("""
WallTime=2.10s
KernelTime=0.61s
UserTime=0.67s
CPUUsage=61%
MaxResidentMemory=61604kB
AverageResidentMemory=0kB
AverageTotalMemory=0kB
AverageUnsharedMemory=0kB
AverageUnsharedStack=0kB
AverageSharedMemory=0kB
PageSize=4096B
MajorPageFaults=0
MinorPageFaults=101799
Swaps=0
ForcedSwitches=82
WaitSwitches=487
Inputs=0
Outputs=400
SocketReceived=0
SocketSent=0
Signals=0
ReturnCode=0
""")
with temporary_core(type='shellcmd') as core:
assert 'test' in core.resources
backend = core.get_backend('test')
termstatus, acctinfo, valid = backend._parse_wrapper_output(resource_usage)
assert valid
assert acctinfo.duration == 2.10*s
assert acctinfo.shellcmd_kernel_time == 0.61*s
assert acctinfo.shellcmd_user_time == 0.67*s
assert acctinfo.max_used_memory == 61604*kB
assert acctinfo.shellcmd_average_total_memory == 0*kB
assert acctinfo.shellcmd_filesystem_inputs == 0
assert acctinfo.shellcmd_filesystem_outputs == 400
assert acctinfo.shellcmd_swapped == 0
assert termstatus == (0, 0)
def test_shellcmd_backend_parse_wrapper_output_signalled():
resource_usage = StringIO("""Command terminated by signal 15
WallTime=6.44s
KernelTime=0.00s
UserTime=0.00s
CPUUsage=0%
MaxResidentMemory=2004kB
AverageResidentMemory=0kB
AverageTotalMemory=0kB
AverageUnsharedMemory=0kB
AverageUnsharedStack=0kB
AverageSharedMemory=0kB
PageSize=4096B
MajorPageFaults=0
MinorPageFaults=73
Swaps=0
ForcedSwitches=0
WaitSwitches=2
Inputs=0
Outputs=0
SocketReceived=0
SocketSent=0
Signals=0
ReturnCode=0
""")
with temporary_core(type='shellcmd') as core:
assert 'test' in core.resources
backend = core.get_backend('test')
termstatus, acctinfo, valid = backend._parse_wrapper_output(resource_usage)
assert valid
assert acctinfo.duration == 6.44*s
assert acctinfo.shellcmd_kernel_time == 0.*s
assert acctinfo.shellcmd_user_time == 0.*s
assert acctinfo.max_used_memory == 2004*kB
assert acctinfo.shellcmd_average_total_memory == 0*kB
assert acctinfo.shellcmd_filesystem_inputs == 0
assert acctinfo.shellcmd_filesystem_outputs == 0
assert acctinfo.shellcmd_swapped == 0
assert termstatus == (15, -1)
def test_staged_task_collection_stage():
class TwoStageCollection(StagedTaskCollection):
def stage0(self):
return SuccessfulApp(name='stage0')
def stage1(self):
return UnsuccessfulApp(name='stage1')
with temporary_core() as core:
coll = TwoStageCollection()
coll.attach(core)
coll.submit()
stage = coll.stage()
assert isinstance(stage, SuccessfulApp), ("stage=%r" % (stage, ))
assert stage.jobname == 'stage0'
# advance to next task
while coll.tasks[0].execution.state != Run.State.TERMINATED:
coll.progress()
stage = coll.stage()
assert isinstance(stage, UnsuccessfulApp)
assert stage.jobname == 'stage1'
def test_SequentialTaskCollection_redo3():
"""Test that we can re-do a partially terminated sequence."""
with temporary_core() as core:
seq = SimpleSequentialTaskCollection(3)
seq.attach(core)
# run until stage1 is terminated
while seq.tasks[1].execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.stage().jobname == 'stage2'
core.kill(seq)
seq.redo(0)
assert seq.stage().jobname == 'stage0'
assert seq.stage().execution.state == Run.State.NEW
assert seq.execution.state == Run.State.NEW
# run until terminated
while seq.execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.stage().jobname == 'stage2'
assert seq.stage().execution.state == Run.State.TERMINATED
def test_SequentialTaskCollection_redo3():
"""Test that we can re-do a partially terminated sequence."""
with temporary_core() as core:
seq = SimpleSequentialTaskCollection(3)
seq.attach(core)
# run until stage1 is terminated
while seq.tasks[1].execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.stage().jobname == 'stage2'
core.kill(seq)
seq.redo(0)
assert seq.stage().jobname == 'stage0'
assert seq.stage().execution.state == Run.State.NEW
assert seq.execution.state == Run.State.NEW
# run until terminated
while seq.execution.state != Run.State.TERMINATED:
seq.progress()
assert seq.stage().jobname == 'stage2'
assert seq.stage().execution.state == Run.State.TERMINATED
def test_task_redo3():
"""Test that `.redo()` is a no-op if the Task is still NEW."""
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.redo()
def test_task_redo3():
"""Test that `.redo()` is a no-op if the Task is still NEW."""
with temporary_core() as core:
task = SuccessfulApp()
task.attach(core)
task.redo()
