def test_staged_task_collection_progress(): class ThreeStageCollection(StagedTaskCollection): def stage0(self): return SuccessfulApp() def stage1(self): return SuccessfulApp() def stage2(self): return UnsuccessfulApp() with temporary_core() as core: coll = ThreeStageCollection() coll.attach(core) coll.submit() assert coll.execution.state == Run.State.SUBMITTED # first task is successful while coll.tasks[0].execution.state != Run.State.TERMINATED: coll.progress() assert coll.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING] #assert_equal(coll.execution.exitcode, 0) # second task is successful while coll.tasks[1].execution.state != Run.State.TERMINATED: coll.progress() #assert_equal(coll.execution.state, Run.State.RUNNING) #assert_equal(coll.execution.exitcode, 0) # third task is unsuccessful while coll.tasks[2].execution.state != Run.State.TERMINATED: coll.progress() assert coll.execution.state == Run.State.TERMINATED assert coll.execution.exitcode == 1
def test_task_state_handlers(): report = { 'submitted': False, 'terminating': False, 'terminated': False, } class HandlerTestApp(SuccessfulApp): def __init__(self, report): super(HandlerTestApp, self).__init__() self.report = report def submitted(self): self.report['submitted'] = True print("Submitted!") def terminating(self): self.report['terminating'] = True print("Terminating!") def terminated(self): print("Terminated!") self.report['terminated'] = True with temporary_core() as core: task = HandlerTestApp(report) task.attach(core) task.submit() assert report['submitted'] # run until terminated while task.execution.state != Run.State.TERMINATED: task.progress() assert report['terminating'] assert report['terminated']
def test_shellcmd_backend_create_spooldir(): with temporary_directory() as tmpdir: with temporary_core(spooldir=tmpdir) as core: assert 'test' in core.resources backend = core.get_backend('test') app = SuccessfulApp() core.submit(app) assert os.path.isdir(backend.spooldir)
def test_ParallelTaskCollection_progress(): with temporary_core(max_cores=10) as core: par = SimpleParallelTaskCollection(5) par.attach(core) # run until terminated while par.execution.state != Run.State.TERMINATED: par.progress() for task in par.tasks: assert task.execution.state == Run.State.TERMINATED
def test_SequentialTaskCollection_progress(): with temporary_core() as core: seq = SimpleSequentialTaskCollection(3) seq.attach(core) # run until terminated while seq.execution.state != Run.State.TERMINATED: seq.progress() assert seq.stage().jobname == 'stage2' assert seq.stage().execution.state == Run.State.TERMINATED
def test_empty_SequentialTaskCollection_progress(): with temporary_core() as core: seq = SimpleSequentialTaskCollection(0) seq.attach(core) # run until terminated while seq.execution.state != Run.State.TERMINATED: seq.progress() assert seq.execution.state == Run.State.TERMINATED assert seq.execution.returncode == 0
def test_task_redo2(): """Test that `.redo()` raises if called on a Task that is not TERMINATED.""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # cannot redo a task that is not yet terminated task.redo()
def test_task_progress(): with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # run until terminated while task.execution.state != Run.State.TERMINATED: task.progress() assert task.execution.state == Run.State.TERMINATED
def test_task_redo2(): """Test that `.redo()` raises if called on a Task that is not TERMINATED.""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # cannot redo a task that is not yet terminated with pytest.raises(AssertionError, message="`Task.redo()` succeeded on task not yet finished"): task.redo()
def test_task_redo2(): """Test that `.redo()` raises if called on a Task that is not TERMINATED.""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # cannot redo a task that is not yet terminated with pytest.raises(AssertionError): task.redo() pytest.fail("`Task.redo()` succeeded on task not yet finished")
def test_core_resources(): """ Check that configured resources can be accessed through the `Core` object. """ with temporary_core() as core: resources = core.resources assert len(resources) == 1 assert 'test' in resources test_rsc = resources['test'] # these should match the resource definition in `gc3libs.testing.helpers.temporary_core` assert test_rsc.max_cores_per_job == 1 assert test_rsc.max_memory_per_core == 1 * GB assert test_rsc.max_walltime == 8 * hours
def test_engine_find_task_by_id(): """ Test that saved tasks are can be retrieved from the Engine given their ID only. """ with temporary_core() as core: with temporary_directory() as tmpdir: store = FilesystemStore(tmpdir) engine = Engine(core, store=store) task = SuccessfulApp() store.save(task) engine.add(task) task_id = task.persistent_id assert engine.find_task_by_id(task_id) == task
def test_engine_cannot_find_task_by_id_if_not_saved(): """ Test that *unsaved* tasks are cannot be retrieved from the Engine given their ID only. """ with temporary_core() as core: with temporary_directory() as tmpdir: store = FilesystemStore(tmpdir) engine = Engine(core, store=store) task = SuccessfulApp() engine.add(task) store.save(task) # guarantee it has a `.persistent_id` task_id = task.persistent_id with pytest.raises(KeyError): engine.find_task_by_id(task_id)
def test_task_redo1(): """Test correct use of `Task.redo()`""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # run until terminated while task.execution.state != Run.State.TERMINATED: task.progress() assert task.execution.state == Run.State.TERMINATED # no do it all over again task.redo() assert task.execution.state == Run.State.NEW task.progress() assert task.execution.state != Run.State.NEW assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
def test_staged_task_collection_stage(): class TwoStageCollection(StagedTaskCollection): def stage0(self): return SuccessfulApp(name='stage0') def stage1(self): return UnsuccessfulApp(name='stage1') with temporary_core() as core: coll = TwoStageCollection() coll.attach(core) coll.submit() stage = coll.stage() assert isinstance(stage, SuccessfulApp), ("stage=%r" % (stage,)) assert stage.jobname == 'stage0' # advance to next task while coll.tasks[0].execution.state != Run.State.TERMINATED: coll.progress() stage = coll.stage() assert isinstance(stage, UnsuccessfulApp) assert stage.jobname == 'stage1'
def test_SequentialTaskCollection_redo2(): with temporary_core() as core: seq = SimpleSequentialTaskCollection(3) seq.attach(core) # run until terminated while seq.execution.state != Run.State.TERMINATED: seq.progress() assert seq.stage().jobname == "stage2" assert seq.stage().execution.state == Run.State.TERMINATED seq.redo(1) assert seq.stage().jobname == "stage1" assert seq.stage().execution.state == Run.State.NEW assert seq.execution.state == Run.State.NEW # run until terminated, again while seq.execution.state != Run.State.TERMINATED: seq.progress() assert seq.stage().jobname == "stage2" assert seq.stage().execution.state == Run.State.TERMINATED
def test_shellcmd_backend_parse_wrapper_output_successful(): resource_usage = StringIO(""" WallTime=2.10s KernelTime=0.61s UserTime=0.67s CPUUsage=61% MaxResidentMemory=61604kB AverageResidentMemory=0kB AverageTotalMemory=0kB AverageUnsharedMemory=0kB AverageUnsharedStack=0kB AverageSharedMemory=0kB PageSize=4096B MajorPageFaults=0 MinorPageFaults=101799 Swaps=0 ForcedSwitches=82 WaitSwitches=487 Inputs=0 Outputs=400 SocketReceived=0 SocketSent=0 Signals=0 ReturnCode=0 """) with temporary_core(type='shellcmd') as core: assert 'test' in core.resources backend = core.get_backend('test') termstatus, acctinfo, valid = backend._parse_wrapper_output(resource_usage) assert valid assert acctinfo.duration == 2.10*s assert acctinfo.shellcmd_kernel_time == 0.61*s assert acctinfo.shellcmd_user_time == 0.67*s assert acctinfo.max_used_memory == 61604*kB assert acctinfo.shellcmd_average_total_memory == 0*kB assert acctinfo.shellcmd_filesystem_inputs == 0 assert acctinfo.shellcmd_filesystem_outputs == 400 assert acctinfo.shellcmd_swapped == 0 assert termstatus == (0, 0)
def test_shellcmd_backend_parse_wrapper_output_signalled(): resource_usage = StringIO("""Command terminated by signal 15 WallTime=6.44s KernelTime=0.00s UserTime=0.00s CPUUsage=0% MaxResidentMemory=2004kB AverageResidentMemory=0kB AverageTotalMemory=0kB AverageUnsharedMemory=0kB AverageUnsharedStack=0kB AverageSharedMemory=0kB PageSize=4096B MajorPageFaults=0 MinorPageFaults=73 Swaps=0 ForcedSwitches=0 WaitSwitches=2 Inputs=0 Outputs=0 SocketReceived=0 SocketSent=0 Signals=0 ReturnCode=0 """) with temporary_core(type='shellcmd') as core: assert 'test' in core.resources backend = core.get_backend('test') termstatus, acctinfo, valid = backend._parse_wrapper_output(resource_usage) assert valid assert acctinfo.duration == 6.44*s assert acctinfo.shellcmd_kernel_time == 0.*s assert acctinfo.shellcmd_user_time == 0.*s assert acctinfo.max_used_memory == 2004*kB assert acctinfo.shellcmd_average_total_memory == 0*kB assert acctinfo.shellcmd_filesystem_inputs == 0 assert acctinfo.shellcmd_filesystem_outputs == 0 assert acctinfo.shellcmd_swapped == 0 assert termstatus == (15, -1)
def test_staged_task_collection_stage(): class TwoStageCollection(StagedTaskCollection): def stage0(self): return SuccessfulApp(name='stage0') def stage1(self): return UnsuccessfulApp(name='stage1') with temporary_core() as core: coll = TwoStageCollection() coll.attach(core) coll.submit() stage = coll.stage() assert isinstance(stage, SuccessfulApp), ("stage=%r" % (stage, )) assert stage.jobname == 'stage0' # advance to next task while coll.tasks[0].execution.state != Run.State.TERMINATED: coll.progress() stage = coll.stage() assert isinstance(stage, UnsuccessfulApp) assert stage.jobname == 'stage1'
def test_SequentialTaskCollection_redo3(): """Test that we can re-do a partially terminated sequence.""" with temporary_core() as core: seq = SimpleSequentialTaskCollection(3) seq.attach(core) # run until stage1 is terminated while seq.tasks[1].execution.state != Run.State.TERMINATED: seq.progress() assert seq.stage().jobname == 'stage2' core.kill(seq) seq.redo(0) assert seq.stage().jobname == 'stage0' assert seq.stage().execution.state == Run.State.NEW assert seq.execution.state == Run.State.NEW # run until terminated while seq.execution.state != Run.State.TERMINATED: seq.progress() assert seq.stage().jobname == 'stage2' assert seq.stage().execution.state == Run.State.TERMINATED
def test_task_redo3(): """Test that `.redo()` is a no-op if the Task is still NEW.""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.redo()