Пример #1
0
def test_engine_kill_SequentialTaskCollection():
    with temporary_engine() as engine:
        seq = SimpleSequentialTaskCollection(3)
        engine.add(seq)

        while seq.execution.state != 'RUNNING':
            engine.progress()

        # When the sequence is in RUNNING state, so must the first app
        assert seq.tasks[0].execution.state == 'RUNNING'
        assert seq.tasks[1].execution.state == 'NEW'
        assert seq.tasks[2].execution.state == 'NEW'

        # Killing a sequential should put all the applications in
        # TERMINATED state. However, we will need an extra run of
        # engine.progress() to update the status of all the jobs.
        engine.kill(seq)
        assert seq.tasks[0].execution.state == 'RUNNING'
        assert seq.tasks[1].execution.state == 'NEW'
        assert seq.tasks[2].execution.state == 'NEW'
        assert seq.execution.state == 'RUNNING'

        engine.progress()

        assert seq.tasks[0].execution.state == 'TERMINATED'
        assert seq.tasks[1].execution.state == 'TERMINATED'
        assert seq.tasks[2].execution.state == 'TERMINATED'
        assert seq.execution.state == 'TERMINATED'
Пример #2
0
def test_engine_forget_terminated(num_jobs=3,
                                  transition_graph=None,
                                  max_iter=100):
    with temporary_engine() as engine:
        engine.forget_terminated = True

        # generate some no-op tasks
        tasks = []
        for n in range(num_jobs):
            name = 'app{nr}'.format(nr=n + 1)
            app = SuccessfulApp(name)
            engine.add(app)
            tasks.append(app)

        # run them all
        current_iter = 0
        done = engine.counts()[Run.State.TERMINATED]
        while done < num_jobs and current_iter < max_iter:
            engine.progress()
            done = engine.counts()[Run.State.TERMINATED]
            current_iter += 1

        # check that they have been forgotten
        assert 0 == len(engine._managed.done)
        for task in tasks:
            assert task.execution.state == 'TERMINATED'
            assert not task._attached
Пример #3
0
def test_engine_kill_ParallelTaskCollection():
    # Creates an engine with 2 cores.
    with temporary_engine(max_cores=2) as engine:
        par = SimpleParallelTaskCollection(3)
        engine.add(par)

        for _ in range(20):
            engine.progress()
            if par.execution.state == 'RUNNING':
                break

        # Because of our noop engine, as soon as the parallel is in
        # running we will have all jobs in SUBMITTED and the others in
        # NEW.
        assert (['SUBMITTED', 'SUBMITTED',
                 'NEW'] == [i.execution.state for i in par.tasks])

        # Killing a parallel should put all the applications in
        # TERMINATED state. However, we need two runs of
        # engine.progress() to update the status of all the jobs
        engine.kill(par)
        assert (['SUBMITTED', 'SUBMITTED',
                 'NEW'] == [i.execution.state for i in par.tasks])

        engine.progress()
        engine.progress()

        assert (['TERMINATED', 'TERMINATED',
                 'TERMINATED'] == [i.execution.state for i in par.tasks])
        assert par.execution.state == 'TERMINATED'
Пример #4
0
def test_engine_kill_ParallelTaskCollection():
    # Creates an engine with 2 cores.
    with temporary_engine(max_cores=2) as engine:
        par = SimpleParallelTaskCollection(3)
        engine.add(par)

        while par.execution.state != 'RUNNING':
            engine.progress()

        # Because of our noop engine, as soon as the parallel is in
        # running we will have all jobs in SUBMITTED and the others in
        # NEW.
        assert_equal(
            ['TERMINATED', 'SUBMITTED', 'NEW'],
            [i.execution.state for i in par.tasks],
        )

        # Killing a parallel should put all the applications in
        # TERMINATED state. However, we need a run of
        # engine.progress() to update the status of all the jobs
        engine.kill(par)
        
        assert_equal(
            ['TERMINATED', 'SUBMITTED', 'NEW'],
            [i.execution.state for i in par.tasks],
        )
        engine.progress()
        
        assert_equal(
            ['TERMINATED', 'TERMINATED', 'TERMINATED'],
            [i.execution.state for i in par.tasks],
        )
        assert_equal(par.execution.state, 'TERMINATED')
Пример #5
0
def test_engine_kill_SequentialTaskCollection():
    with temporary_engine() as engine:
        seq = SimpleSequentialTaskCollection(3)
        engine.add(seq)

        while seq.execution.state != 'RUNNING':
            engine.progress()

        # Because of our noop engine, as soon as the sequential is in
        # running we will have a job in TERMINATED and the others in
        # NEW.
        assert (
            ['TERMINATED', 'NEW', 'NEW'] ==
            [i.execution.state for i in seq.tasks])

        # Killing a sequential should put all the applications in
        # TERMINATED state. However, we will need an extra run of
        # engine.progress() to update the status of all the jobs.
        engine.kill(seq)
        assert (
            ['TERMINATED', 'NEW', 'NEW'] ==
            [i.execution.state for i in seq.tasks])

        engine.progress()

        assert (
            ['TERMINATED', 'TERMINATED', 'TERMINATED'] ==
            [i.execution.state for i in seq.tasks])
        assert seq.execution.state == 'TERMINATED'
Пример #6
0
def test_engine_limits(limit_submitted,
                       limit_in_flight,
                       num_jobs=30,
                       max_iter=100):
    """
    Test that `Engine.limit_in_flight` and `Engine.limit_submitted` are honored.
    """
    with temporary_engine(max_cores=50) as engine:
        # set limits
        engine.max_in_flight = 10
        engine.max_submitted = 2
        # populate with test apps
        apps = []
        for n in range(num_jobs):
            name = 'app{nr}'.format(nr=n)
            app = SuccessfulApp(name)
            engine.add(app)
            apps.append(app)
        # run all apps for (up to) a fixed nr of steps
        iter_nr = 0
        stats = engine.counts()
        while stats['TERMINATED'] < num_jobs and iter_nr < max_iter:
            iter_nr += 1
            engine.progress()
            stats = engine.counts()
            submitted = stats['SUBMITTED']
            assert submitted <= engine.max_submitted
            in_flight = (stats['SUBMITTED'] + stats['RUNNING'])
            assert in_flight <= engine.max_in_flight
        # catch errors in case of termination because of exceeded iter count
        assert stats["TERMINATED"] == num_jobs
Пример #7
0
def test_engine_kill_SequentialTaskCollection():
    with temporary_engine() as engine:
        seq = SimpleSequentialTaskCollection(3)
        engine.add(seq)

        while seq.execution.state != 'RUNNING':
            engine.progress()

        # Because of our noop engine, as soon as the sequential is in
        # running we will have a job in TERMINATED and the others in
        # NEW.
        assert_equal(
            ['TERMINATED', 'NEW', 'NEW'],
            [i.execution.state for i in seq.tasks],
        )

        # Killing a sequential should put all the applications in
        # TERMINATED state. However, we will need an extra run of
        # engine.progress() to update the status of all the jobs.
        engine.kill(seq)
        assert_equal(
            ['TERMINATED', 'NEW', 'NEW'],
            [i.execution.state for i in seq.tasks],
        )

        engine.progress()

        assert_equal(
            ['TERMINATED', 'TERMINATED', 'TERMINATED'],
            [i.execution.state for i in seq.tasks],
        )
        assert_equal(seq.execution.state, 'TERMINATED')
Пример #8
0
def test_engine_progress_collection():
    with temporary_engine() as engine:
        seq = SimpleSequentialTaskCollection(3)
        engine.add(seq)

        # run through sequence
        while seq.execution.state != 'TERMINATED':
            engine.progress()
        assert seq.stage().jobname == 'stage2'
        assert seq.stage().execution.state == 'TERMINATED'
Пример #9
0
def test_engine_progress_collection():
    with temporary_engine() as engine:
        seq = SimpleSequentialTaskCollection(3)
        engine.add(seq)

        # run through sequence
        while seq.execution.state != 'TERMINATED':
            engine.progress()
        assert seq.stage().jobname == 'stage2'
        assert seq.stage().execution.state == 'TERMINATED'
Пример #10
0
def test_engine_redo_Task2():
    """Test that `Engine.redo()` raises if called on a Task that is not TERMINATED."""
    with temporary_engine() as engine:
        task = SuccessfulApp()
        engine.add(task)

        engine.progress()
        assert task.execution.state != Run.State.NEW

        # cannot redo a task that is not yet terminated
        task.redo()
Пример #11
0
def test_engine_redo_ParallelTaskCollection():
    with temporary_engine() as engine:
        par = SimpleParallelTaskCollection(5)
        engine.add(par)

        # run until terminated
        while par.execution.state != Run.State.TERMINATED:
            engine.progress()

        engine.redo(par)
        assert par.execution.state == Run.State.NEW
        for task in par.tasks:
            assert task.execution.state == Run.State.NEW
Пример #12
0
def test_engine_redo_ParallelTaskCollection():
    with temporary_engine() as engine:
        par = SimpleParallelTaskCollection(5)
        engine.add(par)

        # run until terminated
        while par.execution.state != Run.State.TERMINATED:
            engine.progress()

        engine.redo(par)
        assert par.execution.state == Run.State.NEW
        for task in par.tasks:
            assert task.execution.state == Run.State.NEW
Пример #13
0
def test_engine_redo_Task2():
    """Test that `Engine.redo()` raises if called on a Task that is not TERMINATED."""
    with temporary_engine() as engine:
        task = SuccessfulApp()
        engine.add(task)

        engine.progress()
        assert task.execution.state != Run.State.NEW

        # cannot redo a task that is not yet terminated
        with pytest.raises(AssertionError):
            task.redo()
            pytest.fail("`Task.redo()` succeeded on task not yet finished")
Пример #14
0
def test_engine_resources():
    """
    Check that configured resources can be accessed through the `Engine` object.
    """
    with temporary_engine() as engine:
        resources = engine.resources
        assert len(resources) == 1
        assert 'test' in resources
        test_rsc = resources['test']
        # these should match the resource definition in `gc3libs.testing.helpers.temporary_core`
        assert test_rsc.max_cores_per_job == 123
        assert test_rsc.max_memory_per_core == 999 * GB
        assert test_rsc.max_walltime == 7 * hours
Пример #15
0
def test_engine_submit1():
    """Engine.submit is equivalent to `add` if a task is not yet managed."""
    with temporary_engine() as engine:
        assert engine.counts()['NEW'] == 0

        app = SuccessfulApp()
        assert app.execution.state == 'NEW'
        engine.submit(app)
        assert app.execution.state == 'NEW'
        assert engine.counts()['NEW'] == 1

        engine.progress()
        assert app.execution.state in ['SUBMITTED', 'RUNNING']
        assert engine.counts()['NEW'] == 0
        assert engine.counts()[app.execution.state] == 1
Пример #16
0
def test_engine_cannot_find_task_by_id_if_no_store():
    """
    Test that `Engine.find_task_by_id` always raises `KeyError` if the Engine has no associated store.
    """
    with temporary_engine() as engine:
        with temporary_directory() as tmpdir:
            store = FilesystemStore(tmpdir)

            task = SuccessfulApp()
            engine.add(task)

            store.save(task)  # guarantee it has a `.persistent_id`
            task_id = task.persistent_id
            with pytest.raises(KeyError):
                engine.find_task_by_id(task_id)
Пример #17
0
def test_engine_submit1():
    """Engine.submit is equivalent to `add` if a task is not yet managed."""
    with temporary_engine() as engine:
        assert engine.stats()['NEW'] == 0

        app = SuccessfulApp()
        assert app.execution.state == 'NEW'
        engine.submit(app)
        assert app.execution.state == 'NEW'
        assert engine.stats()['NEW'] == 1

        engine.progress()
        assert app.execution.state in ['SUBMITTED', 'RUNNING']
        assert engine.stats()['NEW'] == 0
        assert engine.stats()[app.execution.state] == 1
Пример #18
0
def test_engine_progress_collection_and_forget_terminated():
    with temporary_engine() as engine:
        engine.forget_terminated = True

        seq = SimpleSequentialTaskCollection(3)
        engine.add(seq)

        # run through sequence
        while seq.execution.state != 'TERMINATED':
            engine.progress()

        assert 0 == len(engine._managed.done)
        assert not seq._attached
        for task in seq.tasks:
            assert not task._attached
Пример #19
0
def test_engine_progress(num_jobs=1, transition_graph=None, max_iter=100):
    with temporary_engine() as engine:

        # generate some no-op tasks
        for n in range(num_jobs):
            name = 'app{nr}'.format(nr=n+1)
            engine.add(SuccessfulApp(name))

        # run them all
        current_iter = 0
        done = engine.stats()[Run.State.TERMINATED]
        while done < num_jobs and current_iter < max_iter:
            engine.progress()
            done = engine.stats()[Run.State.TERMINATED]
            current_iter += 1
Пример #20
0
def test_engine_progress(num_jobs=1, transition_graph=None, max_iter=100):
    with temporary_engine() as engine:

        # generate some no-op tasks
        for n in range(num_jobs):
            name = 'app{nr}'.format(nr=n+1)
            engine.add(SuccessfulApp(name))

        # run them all
        current_iter = 0
        done = engine.stats()[Run.State.TERMINATED]
        while done < num_jobs and current_iter < max_iter:
            engine.progress()
            done = engine.stats()[Run.State.TERMINATED]
            current_iter += 1
Пример #21
0
def test_engine_resubmit():
    with temporary_engine() as engine:
        app = SuccessfulApp()
        engine.add(app)

        # run through sequence
        while app.execution.state != 'TERMINATED':
            engine.progress()

        engine.submit(app, resubmit=True)
        assert app.execution.state == 'NEW'

        # run through sequence again
        while app.execution.state != 'TERMINATED':
            engine.progress()
        assert app.execution.state == 'TERMINATED'
Пример #22
0
def test_engine_resubmit():
    with temporary_engine() as engine:
        app = SuccessfulApp()
        engine.add(app)

        # run through sequence
        while app.execution.state != 'TERMINATED':
            engine.progress()

        engine.submit(app, resubmit=True)
        assert app.execution.state == 'NEW'

        # run through sequence again
        while app.execution.state != 'TERMINATED':
            engine.progress()
        assert app.execution.state == 'TERMINATED'
Пример #23
0
def test_engine_redo_Task1():
    """Test correct use of `Engine.redo()` with a `Task` instance."""
    with temporary_engine() as engine:
        task = SuccessfulApp()
        engine.add(task)

        # run until terminated
        while task.execution.state != Run.State.TERMINATED:
            engine.progress()
        assert task.execution.state == Run.State.TERMINATED

        # no do it all over again
        engine.redo(task)
        assert task.execution.state == Run.State.NEW

        engine.progress()
        assert task.execution.state != Run.State.NEW
        assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
Пример #24
0
def test_engine_redo_Task1():
    """Test correct use of `Engine.redo()` with a `Task` instance."""
    with temporary_engine() as engine:
        task = SuccessfulApp()
        engine.add(task)

        # run until terminated
        while task.execution.state != Run.State.TERMINATED:
            engine.progress()
        assert task.execution.state == Run.State.TERMINATED

        # no do it all over again
        engine.redo(task)
        assert task.execution.state == Run.State.NEW

        engine.progress()
        assert task.execution.state != Run.State.NEW
        assert task.execution.state in [Run.State.SUBMITTED, Run.State.RUNNING]
Пример #25
0
def test_engine_submit2():
    """Engine.submit is a no-op if a task is already managed."""
    with temporary_engine() as engine:
        app = SuccessfulApp()
        engine.add(app)
        assert engine.stats()['NEW'] == 1

        engine.submit(app)
        assert engine.stats()['NEW'] == 1

        engine.progress()
        state = app.execution.state
        assert state in ['SUBMITTED', 'RUNNING']
        assert engine.stats()['NEW'] == 0
        assert engine.stats()[state] == 1

        engine.submit(app)
        assert app.execution.state == state
        assert engine.stats()[state] == 1
Пример #26
0
def test_engine_submit2():
    """Engine.submit is a no-op if a task is already managed."""
    with temporary_engine() as engine:
        app = SuccessfulApp()
        engine.add(app)
        assert engine.counts()['NEW'] == 1

        engine.submit(app)
        assert engine.counts()['NEW'] == 1

        engine.progress()
        state = app.execution.state
        assert state in ['SUBMITTED', 'RUNNING']
        assert engine.counts()['NEW'] == 0
        assert engine.counts()[state] == 1

        engine.submit(app)
        assert app.execution.state == state
        assert engine.counts()[state] == 1
Пример #27
0
def test_engine_progress(num_jobs=5, transition_graph=None, max_iter=100):
    with temporary_engine() as engine:

        # generate some no-op tasks
        tasks = []
        for n in range(num_jobs):
            name = 'app{nr}'.format(nr=n + 1)
            app = SuccessfulApp(name)
            engine.add(app)
            tasks.append(app)

        # run them all
        current_iter = 0
        done = engine.counts()[Run.State.TERMINATED]
        while done < num_jobs and current_iter < max_iter:
            engine.progress()
            done = engine.counts()[Run.State.TERMINATED]
            current_iter += 1

        # check state
        for task in tasks:
            assert task.execution.state == 'TERMINATED'
Пример #28
0
def _test_engine_counts(populate,
                        max_cores,
                        num_jobs,
                        num_new_jobs=None,
                        max_iter=100):
    """
    Common code for the `test_engine_counts*` tests.
    """
    # make the `.progress()` call a little less deterministic
    transition_graph = {
        Run.State.SUBMITTED: {
            0.8: Run.State.RUNNING
        },
        Run.State.RUNNING: {
            0.8: Run.State.TERMINATING
        },
        Run.State.TERMINATING: {
            0.8: Run.State.TERMINATED
        },
    }
    with temporary_engine(transition_graph, max_cores=max_cores) as engine:
        # populate with test apps
        apps = populate(engine)

        # initial check on stats
        actual_counts = engine.counts()
        if num_new_jobs is None:
            num_new_jobs = num_jobs
        assert actual_counts['NEW'] == num_new_jobs

        # check
        iter_nr = 0
        while (actual_counts['TERMINATED'] < num_jobs and iter_nr < max_iter):
            iter_nr += 1
            engine.progress()
            actual_counts = engine.counts()
            expected_counts = _compute_counts(apps)
            _check_counts(actual_counts, expected_counts)
Пример #29
0
def test_engine_redo_Task3():
    """Test that `Engine.redo()` is a no-op if the Task is still NEW."""
    with temporary_engine() as engine:
        task = SuccessfulApp()
        engine.add(task)
        task.redo()
Пример #30
0
def test_engine_redo_Task3():
    """Test that `Engine.redo()` is a no-op if the Task is still NEW."""
    with temporary_engine() as engine:
        task = SuccessfulApp()
        engine.add(task)
        task.redo()