def test_engine_redo_Task2(): """Test that `Engine.redo()` raises if called on a Task that is not TERMINATED.""" with temporary_engine() as engine: task = SuccessfulApp() engine.add(task) engine.progress() assert task.execution.state != Run.State.NEW # cannot redo a task that is not yet terminated task.redo()
def test_engine_redo_Task2(): """Test that `Engine.redo()` raises if called on a Task that is not TERMINATED.""" with temporary_engine() as engine: task = SuccessfulApp() engine.add(task) engine.progress() assert task.execution.state != Run.State.NEW # cannot redo a task that is not yet terminated with pytest.raises(AssertionError): task.redo() pytest.fail("`Task.redo()` succeeded on task not yet finished")
def test_engine_limits(limit_submitted, limit_in_flight, num_jobs=30, max_iter=100): """ Test that `Engine.limit_in_flight` and `Engine.limit_submitted` are honored. """ with temporary_engine(max_cores=50) as engine: # set limits engine.max_in_flight = 10 engine.max_submitted = 2 # populate with test apps apps = [] for n in range(num_jobs): name = 'app{nr}'.format(nr=n) app = SuccessfulApp(name) engine.add(app) apps.append(app) # run all apps for (up to) a fixed nr of steps iter_nr = 0 stats = engine.counts() while stats['TERMINATED'] < num_jobs and iter_nr < max_iter: iter_nr += 1 engine.progress() stats = engine.counts() submitted = stats['SUBMITTED'] assert submitted <= engine.max_submitted in_flight = (stats['SUBMITTED'] + stats['RUNNING']) assert in_flight <= engine.max_in_flight # catch errors in case of termination because of exceeded iter count assert stats["TERMINATED"] == num_jobs
def test_engine_forget_terminated(num_jobs=3, transition_graph=None, max_iter=100): with temporary_engine() as engine: engine.forget_terminated = True # generate some no-op tasks tasks = [] for n in range(num_jobs): name = 'app{nr}'.format(nr=n + 1) app = SuccessfulApp(name) engine.add(app) tasks.append(app) # run them all current_iter = 0 done = engine.counts()[Run.State.TERMINATED] while done < num_jobs and current_iter < max_iter: engine.progress() done = engine.counts()[Run.State.TERMINATED] current_iter += 1 # check that they have been forgotten assert 0 == len(engine._managed.done) for task in tasks: assert task.execution.state == 'TERMINATED' assert not task._attached
def populate(engine): apps = [] for n in range(num_jobs): name = 'app{nr}'.format(nr=n) app = SuccessfulApp(name) apps.append(app) engine.add(app) return apps
def test_shellcmd_backend_create_spooldir(): with temporary_directory() as tmpdir: with temporary_core(spooldir=tmpdir) as core: assert 'test' in core.resources backend = core.get_backend('test') app = SuccessfulApp() core.submit(app) assert os.path.isdir(backend.spooldir)
def test_task_progress(): with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # run until terminated while task.execution.state != Run.State.TERMINATED: task.progress() assert task.execution.state == Run.State.TERMINATED
def test_task_redo2(): """Test that `.redo()` raises if called on a Task that is not TERMINATED.""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # cannot redo a task that is not yet terminated with pytest.raises(AssertionError): task.redo() pytest.fail("`Task.redo()` succeeded on task not yet finished")
def test_task_redo2(): """Test that `.redo()` raises if called on a Task that is not TERMINATED.""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # cannot redo a task that is not yet terminated task.redo()
def test_engine_submit_to_multiple_resources(num_resources=3, num_jobs=50): """Test job spread across multiple resources.""" # sanity check for parameters assert num_jobs > 10*(num_resources*(num_resources-1)/2), \ "There must be enough jobs to fill the first N-1 resources" assert num_jobs < 10*(num_resources*(num_resources+1)/2), \ "Too many jobs: would fill all resources" # set up cfg = gc3libs.config.Configuration() cfg.TYPE_CONSTRUCTOR_MAP['noop'] = ('gc3libs.backends.noop', 'NoOpLrms') for n in range(num_resources): name = 'test{nr}'.format(nr=n + 1) cfg.resources[name].update( name=name, type='noop', auth='none', transport='local', max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=8 * hours, max_cores=((n + 1) * 10), architecture=Run.Arch.X86_64, ) core = Core(cfg) engine = Engine(core) # generate 50 no-op tasks for n in range(num_jobs): name = 'app{nr}'.format(nr=n) engine.add(SuccessfulApp(name)) # submit them all engine.progress() # get handles to the actual backend objects rscs = [ core.get_backend('test{nr}'.format(nr=n + 1)) for n in range(num_resources) ] num_jobs_per_resource = [ len([ task for task in engine._managed.to_update if task.execution.resource_name == rsc.name ]) for rsc in rscs ] # check that all jobs have been submitted and that each # resource got at least one job assert sum(num_jobs_per_resource) == num_jobs for num in num_jobs_per_resource: assert num > 0 # since TYPE_CONSTRUCTOR_MAP is a class-level variable, we # need to clean up otherwise other tests will see the No-Op # backend del cfg.TYPE_CONSTRUCTOR_MAP['noop']
def test_task_redo2(): """Test that `.redo()` raises if called on a Task that is not TERMINATED.""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # cannot redo a task that is not yet terminated with pytest.raises(AssertionError, message="`Task.redo()` succeeded on task not yet finished"): task.redo()
def test_engine_submit1(): """Engine.submit is equivalent to `add` if a task is not yet managed.""" with temporary_engine() as engine: assert engine.counts()['NEW'] == 0 app = SuccessfulApp() assert app.execution.state == 'NEW' engine.submit(app) assert app.execution.state == 'NEW' assert engine.counts()['NEW'] == 1 engine.progress() assert app.execution.state in ['SUBMITTED', 'RUNNING'] assert engine.counts()['NEW'] == 0 assert engine.counts()[app.execution.state] == 1
def test_engine_find_task_by_id(): """ Test that saved tasks are can be retrieved from the Engine given their ID only. """ with temporary_core() as core: with temporary_directory() as tmpdir: store = FilesystemStore(tmpdir) engine = Engine(core, store=store) task = SuccessfulApp() store.save(task) engine.add(task) task_id = task.persistent_id assert engine.find_task_by_id(task_id) == task
def test_engine_cannot_find_task_by_id_if_no_store(): """ Test that `Engine.find_task_by_id` always raises `KeyError` if the Engine has no associated store. """ with temporary_engine() as engine: with temporary_directory() as tmpdir: store = FilesystemStore(tmpdir) task = SuccessfulApp() engine.add(task) store.save(task) # guarantee it has a `.persistent_id` task_id = task.persistent_id with pytest.raises(KeyError): engine.find_task_by_id(task_id)
def test_engine_progress(num_jobs=1, transition_graph=None, max_iter=100): with temporary_engine() as engine: # generate some no-op tasks for n in range(num_jobs): name = 'app{nr}'.format(nr=n+1) engine.add(SuccessfulApp(name)) # run them all current_iter = 0 done = engine.stats()[Run.State.TERMINATED] while done < num_jobs and current_iter < max_iter: engine.progress() done = engine.stats()[Run.State.TERMINATED] current_iter += 1
def test_engine_resubmit(): with temporary_engine() as engine: app = SuccessfulApp() engine.add(app) # run through sequence while app.execution.state != 'TERMINATED': engine.progress() engine.submit(app, resubmit=True) assert app.execution.state == 'NEW' # run through sequence again while app.execution.state != 'TERMINATED': engine.progress() assert app.execution.state == 'TERMINATED'
def test_engine_cannot_find_task_by_id_if_not_saved(): """ Test that *unsaved* tasks are cannot be retrieved from the Engine given their ID only. """ with temporary_core() as core: with temporary_directory() as tmpdir: store = FilesystemStore(tmpdir) engine = Engine(core, store=store) task = SuccessfulApp() engine.add(task) store.save(task) # guarantee it has a `.persistent_id` task_id = task.persistent_id with pytest.raises(KeyError): engine.find_task_by_id(task_id)
def test_engine_redo_Task1(): """Test correct use of `Engine.redo()` with a `Task` instance.""" with temporary_engine() as engine: task = SuccessfulApp() engine.add(task) # run until terminated while task.execution.state != Run.State.TERMINATED: engine.progress() assert task.execution.state == Run.State.TERMINATED # no do it all over again engine.redo(task) assert task.execution.state == Run.State.NEW engine.progress() assert task.execution.state != Run.State.NEW assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
def test_engine_submit2(): """Engine.submit is a no-op if a task is already managed.""" with temporary_engine() as engine: app = SuccessfulApp() engine.add(app) assert engine.counts()['NEW'] == 1 engine.submit(app) assert engine.counts()['NEW'] == 1 engine.progress() state = app.execution.state assert state in ['SUBMITTED', 'RUNNING'] assert engine.counts()['NEW'] == 0 assert engine.counts()[state] == 1 engine.submit(app) assert app.execution.state == state assert engine.counts()[state] == 1
def test_engine_progress(num_jobs=5, transition_graph=None, max_iter=100): with temporary_engine() as engine: # generate some no-op tasks tasks = [] for n in range(num_jobs): name = 'app{nr}'.format(nr=n + 1) app = SuccessfulApp(name) engine.add(app) tasks.append(app) # run them all current_iter = 0 done = engine.counts()[Run.State.TERMINATED] while done < num_jobs and current_iter < max_iter: engine.progress() done = engine.counts()[Run.State.TERMINATED] current_iter += 1 # check state for task in tasks: assert task.execution.state == 'TERMINATED'
def test_task_redo3(): """Test that `.redo()` is a no-op if the Task is still NEW.""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.redo()
def test_task_redo1(): """Test correct use of `Task.redo()`""" with temporary_core() as core: task = SuccessfulApp() task.attach(core) task.submit() assert task.execution.state == Run.State.SUBMITTED # run until terminated while task.execution.state != Run.State.TERMINATED: task.progress() assert task.execution.state == Run.State.TERMINATED # no do it all over again task.redo() assert task.execution.state == Run.State.NEW task.progress() assert task.execution.state != Run.State.NEW assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
def stage1(self): return SuccessfulApp()
def stage0(self): return SuccessfulApp(name='stage0')
def test_engine_redo_Task3(): """Test that `Engine.redo()` is a no-op if the Task is still NEW.""" with temporary_engine() as engine: task = SuccessfulApp() engine.add(task) task.redo()