示例#1
0
def test_retry_failed_task(
        app1, job_id1, job_id2, log, tasks_json_tmpfile):
    """
    Retry failed tasks up to max num retries and then remove self from queue

    Tasks should maintain proper task state throughout.
    """
    # create 2 tasks in same queue
    enqueue(app1, job_id1)
    enqueue(app1, job_id2, validate_queued=False)
    nose.tools.assert_equal(2, get_qb_status(app1, job_id1)['app_qsize'])
    nose.tools.assert_equal(job_id1, cycle_queue(app1))
    # run job_id2 and have it fail
    run_code(
        log, tasks_json_tmpfile, app1,
        extra_opts='--bash_cmd "&& notacommand...fail" ')
    # ensure we still have both items in the queue
    nose.tools.assert_true(get_qb_status(app1, job_id1)['in_queue'])
    nose.tools.assert_true(get_qb_status(app1, job_id2)['in_queue'])
    # ensure the failed task is sent to back of the queue
    nose.tools.assert_equal(2, get_qb_status(app1, job_id1)['app_qsize'])
    nose.tools.assert_equal(job_id1, cycle_queue(app1))
    # run and fail n times, where n = max failures
    run_code(
        log, tasks_json_tmpfile, app1,
        extra_opts='--max_retry 1 --bash_cmd "&& notacommand...fail"')
    # verify that job_id2 is removed from queue
    validate_one_queued_task(app1, job_id1)
    # verify that job_id2 state is 'failed' and job_id1 is still pending
    validate_one_failed_task(app1, job_id2)
示例#2
0
def test_pull_tasks1(app1, app2, job_id1, log, tasks_json_tmpfile):
    """
    Parent tasks should be generated and executed before child tasks
    (The Bubble Up and then Bubble Down test)

    If A --> B, and:
        we queue and run B, then we should have 0 completed tasks,
        but A should be queued

        nothing should change until:
            we run A and A becomes completed
            we then run B and B becomes completed
    """
    enqueue(app2, job_id1)
    run_code(log, tasks_json_tmpfile, app2)
    validate_one_queued_task(app1, job_id1)
    validate_zero_queued_task(app2)

    run_code(log, tasks_json_tmpfile, app2)
    validate_one_queued_task(app1, job_id1)
    validate_zero_queued_task(app2)

    run_code(log, tasks_json_tmpfile, app1)
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    run_code(log, tasks_json_tmpfile, app2)
    validate_one_completed_task(app2, job_id1)
示例#3
0
def test_rerun_push_tasks1(app1, app2, job_id1):
    # this tests recursively deleteing parent status on child nodes

    # queue and complete app 1. it queues a child
    enqueue(app1, job_id1)
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)

    # complete app 2
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)

    # readd app 1
    qb.readd_subtask(app1, job_id1)
    validate_one_queued_task(app1, job_id1)
    validate_zero_queued_task(app2)
    nose.tools.assert_true(
        qb.check_state(app2, job_id1, pending=True))

    # complete app 1
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    # complete app 2
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
示例#4
0
def test_valid_if_or_func2(app3, job_id1, job_id2, job_id3):
    """Verify that the valid_if_or option supports the "_func" option

    app_name: {"valid_if_or": {"_func": "python.import.path.to.func"}}
    where the function definition looks like: func(**parsed_job_id)
    """
    enqueue(app3, job_id3, validate_queued=False)
    validate_one_skipped_task(app3, job_id3)
示例#5
0
def test_valid_if_or_func3(app3, job_id1, job_id2, job_id3):
    """Verify that the valid_if_or option supports the "_func" option

    app_name: {"valid_if_or": {"_func": "python.import.path.to.func"}}
    where the function definition looks like: func(**parsed_job_id)
    """
    # if the job_id matches the valid_if_or: {"_func": func...} criteria, then:
    enqueue(app3, job_id1, validate_queued=False)
    validate_one_queued_task(app3, job_id1)
示例#6
0
def test_run_given_specific_job_id(app1, job_id1, log, tasks_json_tmpfile):
    enqueue(app1, job_id1)
    out, err = run_code(
        log, tasks_json_tmpfile, app1,
        '--job_id %s' % job_id1, raise_on_err=False, capture=True)
    nose.tools.assert_regexp_matches(err, (
        'UserWarning: Will not execute this task because it might be'
        ' already queued or completed!'))
    validate_one_queued_task(app1, job_id1)
示例#7
0
def test_app_has_command_line_params(
        bash1, job_id1, log, tasks_json_tmpfile):
    enqueue(bash1, job_id1)
    msg = 'output: %s'
    # Test passed in params exist
    _, logoutput = run_code(
        log, tasks_json_tmpfile, bash1,
        extra_opts='--redirect_to_stderr --bash_cmd echo newfakereadfp',
        capture=True, raise_on_err=True)
    nose.tools.assert_in(
        'newfakereadfp', logoutput, msg % logoutput)
示例#8
0
def test_push_tasks1(app1, app2, job_id1, log, tasks_json_tmpfile):
    """
    Child tasks should be generated and executed properly

    if task A --> task B, and we queue & run A,
    then we should end up with one A task completed and one queued B task
    """
    enqueue(app1, job_id1)
    run_code(log, tasks_json_tmpfile, app1)
    validate_one_completed_task(app1, job_id1)
    # check child
    validate_one_queued_task(app2, job_id1)
示例#9
0
def test_bash1(bash1, job_id1, log, tasks_json_tmpfile):
    """a bash task should execute properly """
    # queue task
    enqueue(bash1, job_id1)
    validate_one_queued_task(bash1, job_id1)
    # run failing task
    run_code(
        log, tasks_json_tmpfile, bash1, '--bash_cmd thiscommandshouldfail')
    validate_one_queued_task(bash1, job_id1)
    # run successful task
    run_code(log, tasks_json_tmpfile, bash1, '--bash_cmd echo 123')
    validate_zero_queued_task(bash1)
示例#10
0
文件: test_api.py 项目: kszucs/stolos
def test_get_qsize(app1, job_id1, job_id2):
    nt.assert_equal(api.get_qsize(app1, queued=True, taken=True), 0)
    tt.enqueue(app1, job_id1, )
    tt.enqueue(app1, job_id2, validate_queued=False)
    q = api.get_qbclient().LockingQueue(app1)
    itm = q.get()
    nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=True))
    nt.assert_equal(1, api.get_qsize(app1, queued=False, taken=True))
    nt.assert_equal(1, api.get_qsize(app1, queued=True, taken=False))
    q.put(itm)
    q.consume()
    nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=True))
    nt.assert_equal(0, api.get_qsize(app1, queued=False, taken=True))
    nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=False))
示例#11
0
def test_skipped_parent_and_queued_child(app1, app2, app3, app4, job_id1,
                                         log, tasks_json_tmpfile):
    qb.set_state(app1, job_id1, skipped=True)
    qb.set_state(app3, job_id1, skipped=True)
    qb.set_state(app2, job_id1, skipped=True)
    enqueue(app4, job_id1)
    validate_zero_queued_task(app2)
    validate_one_queued_task(app4, job_id1)
    # ensure child unqueues itself and raises warning
    out, err = run_code(log, tasks_json_tmpfile, app4, capture=True)
    nose.tools.assert_in(
        "parent_job_id is marked as 'skipped', so should be impossible for me,"
        " the child, to exist", err)
    validate_zero_queued_task(app4)
    validate_zero_queued_task(app2)
示例#12
0
def test_complex_dependencies_pull_push(
        depends_on1, depends_on_job_id1, log, tasks_json_tmpfile):
    job_id = depends_on_job_id1
    enqueue(depends_on1, job_id)
    run_code(log, tasks_json_tmpfile, depends_on1, '--bash_cmd echo 123')

    parents = api.get_parents(depends_on1, job_id)
    parents = list(api.topological_sort(parents))
    for parent, pjob_id in parents[:-1]:
        qb.set_state(parent, pjob_id, completed=True)
        validate_zero_queued_task(depends_on1)
    qb.set_state(*parents[-1], completed=True)
    validate_one_queued_task(depends_on1, job_id)
    run_code(log, tasks_json_tmpfile, depends_on1, '--bash_cmd echo 123')
    validate_one_completed_task(depends_on1, job_id)
示例#13
0
def test_child_running_while_parent_pending_but_not_executing(
        app1, app2, job_id1):
    enqueue(app1, job_id1)
    enqueue(app2, job_id1)
    parents_completed, consume_queue, parent_lock = \
        qb.ensure_parents_completed(app2, job_id1)
    # ensure lock is obtained by ensure_parents_completed
    validate_one_queued_executing_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    nose.tools.assert_equal(parents_completed, False)
    # child should promise to remove itself from queue
    nose.tools.assert_equal(consume_queue, True)
    nose.tools.assert_is_instance(parent_lock, qb.BaseLock)
    # cleanup
    parent_lock.release()
示例#14
0
def test_child_running_while_parent_pending_and_executing(
        app1, app2, job_id1):
    enqueue(app1, job_id1)
    enqueue(app2, job_id1)
    lock = qb.obtain_execute_lock(app1, job_id1)
    assert lock
    parents_completed, consume_queue, parent_lock = \
        qb.ensure_parents_completed(app2, job_id1)
    validate_one_queued_executing_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    nose.tools.assert_equal(parents_completed, False)
    # child should not promise to remove itself from queue
    nose.tools.assert_equal(consume_queue, False)
    nose.tools.assert_is_none(parent_lock)
    # cleanup
    lock.release()
示例#15
0
def test_get_qsize(app1, job_id1, job_id2):
    nt.assert_equal(api.get_qsize(app1, queued=True, taken=True), 0)
    tt.enqueue(
        app1,
        job_id1,
    )
    tt.enqueue(app1, job_id2, validate_queued=False)
    q = api.get_qbclient().LockingQueue(app1)
    itm = q.get()
    nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=True))
    nt.assert_equal(1, api.get_qsize(app1, queued=False, taken=True))
    nt.assert_equal(1, api.get_qsize(app1, queued=True, taken=False))
    q.put(itm)
    q.consume()
    nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=True))
    nt.assert_equal(0, api.get_qsize(app1, queued=False, taken=True))
    nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=False))
示例#16
0
def test_should_not_add_queue_while_consuming_queue(app1, job_id1):
    # This test guards from doubly queuing jobs
    # This protects from simultaneous operations on root and leaf nodes
    # ie (parent and child) for the following operations:
    # adding, readding or a mix of both
    enqueue(app1, job_id1)

    q = qb.get_qbclient().LockingQueue(app1)
    q.get()
    validate_one_queued_task(app1, job_id1)

    enqueue(app1, job_id1)
    with nose.tools.assert_raises(exceptions.JobAlreadyQueued):
        qb.readd_subtask(app1, job_id1)
    validate_one_queued_task(app1, job_id1)

    # cleanup
    q.consume()
示例#17
0
def test_rerun_pull_tasks1(app1, app2, job_id1, log, tasks_json_tmpfile):
    # queue and complete app 1. it queues a child
    enqueue(app1, job_id1)
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    validate_zero_queued_task(app1)
    validate_one_queued_task(app2, job_id1)
    # complete app 2
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_zero_queued_task(app2)

    # readd app 2
    qb.readd_subtask(app2, job_id1)
    validate_zero_queued_task(app1)
    validate_one_queued_task(app2, job_id1)
    # run app 2.  the parent was previously completed
    run_code(log, tasks_json_tmpfile, app2)
    validate_one_completed_task(app1, job_id1)  # previously completed
    validate_one_completed_task(app2, job_id1)
示例#18
0
def _test_rerun_tasks_when_manually_queuing_child_and_parent(
        app1, app2, job_id1):
    # complete parent and child
    enqueue(app1, job_id1)
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)

    # manually re-add child
    qb.readd_subtask(app2, job_id1)
    validate_one_queued_task(app2, job_id1)
    validate_one_completed_task(app1, job_id1)

    # manually re-add parent
    qb.readd_subtask(app1, job_id1)
    validate_one_queued_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
示例#19
0
def test_pull_tasks_with_many_children(app1, app2, app3, app4, job_id1,
                                       log, tasks_json_tmpfile):
    enqueue(app4, job_id1)
    validate_one_queued_task(app4, job_id1)
    validate_zero_queued_task(app1)
    validate_zero_queued_task(app2)
    validate_zero_queued_task(app3)

    run_code(log, tasks_json_tmpfile, app4, '--bash_cmd echo app4helloworld')
    validate_zero_queued_task(app4)
    validate_one_queued_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    validate_one_queued_task(app3, job_id1)

    consume_queue(app1)
    qb.set_state(app1, job_id1, completed=True)
    validate_zero_queued_task(app4)
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    validate_one_queued_task(app3, job_id1)

    consume_queue(app2)
    qb.set_state(app2, job_id1, completed=True)
    validate_zero_queued_task(app4)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
    validate_one_queued_task(app3, job_id1)

    consume_queue(app3)
    qb.set_state(app3, job_id1, completed=True)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
    validate_one_completed_task(app3, job_id1)
    validate_one_queued_task(app4, job_id1)

    consume_queue(app4)
    qb.set_state(app4, job_id1, completed=True)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
    validate_one_completed_task(app3, job_id1)
    validate_one_completed_task(app4, job_id1)
示例#20
0
def test_rerun_manual_task1(app1, job_id1):
    enqueue(app1, job_id1)
    validate_one_queued_task(app1, job_id1)

    with nose.tools.assert_raises(exceptions.JobAlreadyQueued):
        qb.readd_subtask(app1, job_id1)
示例#21
0
def test_valid_if_or_1(app2, job_id1):
    """Invalid tasks should be automatically completed.
    This is a valid_if_or test  (aka passes_filter )... bad naming sorry!"""
    job_id = job_id1.replace('profile', 'content')
    enqueue(app2, job_id, validate_queued=False)
    validate_one_skipped_task(app2, job_id)
示例#22
0
def test_valid_task1(app2, job_id1):
    """Valid tasks should be automatically completed"""
    enqueue(app2, job_id1, validate_queued=True)