Ejemplo n.º 1
0
def test_create_child_task_after_one_parent_completed(
        app1, app2, app3, job_id1, log, tasks_json_tmpfile, func_name):
    # if you modify the tasks.json file in the middle of processing the dag
    # modifications to the json file should be recognized

    # the child task should run if another parent completes
    # but otherwise should not run until it's manually queued

    injected_app = app3
    dct = {
        injected_app: {
            "depends_on": {"app_name": [app1, app2]},
        },
    }

    qb.set_state(
        app1, job_id1, completed=True)
    validate_one_completed_task(app1, job_id1)
    validate_zero_queued_task(injected_app)
    consume_queue(app2)
    qb.set_state(app2, job_id1, completed=True)

    with inject_into_dag(func_name, dct):
        validate_zero_queued_task(injected_app)
        qb.set_state(app2, job_id1, completed=True)
        validate_zero_queued_task(injected_app)
        qb.set_state(app1, job_id1, completed=True)
        validate_one_queued_task(injected_app, job_id1)

        run_code(log, tasks_json_tmpfile, injected_app, '--bash_cmd echo 123')
        validate_one_completed_task(injected_app, job_id1)
Ejemplo n.º 2
0
def test_rerun_push_tasks1(app1, app2, job_id1):
    # this tests recursively deleteing parent status on child nodes

    # queue and complete app 1. it queues a child
    enqueue(app1, job_id1)
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)

    # complete app 2
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)

    # readd app 1
    qb.readd_subtask(app1, job_id1)
    validate_one_queued_task(app1, job_id1)
    validate_zero_queued_task(app2)
    nose.tools.assert_true(
        qb.check_state(app2, job_id1, pending=True))

    # complete app 1
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    # complete app 2
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
Ejemplo n.º 3
0
def test_rerun_push_tasks_when_manually_queuing_child_and_parent(
        app1, app2, job_id1):
    _test_rerun_tasks_when_manually_queuing_child_and_parent(
        app1, app2, job_id1)

    # complete parent first
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)

    # child completes normally
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
Ejemplo n.º 4
0
def _test_rerun_tasks_when_manually_queuing_child_and_parent(
        app1, app2, job_id1):
    # complete parent and child
    enqueue(app1, job_id1)
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)

    # manually re-add child
    qb.readd_subtask(app2, job_id1)
    validate_one_queued_task(app2, job_id1)
    validate_one_completed_task(app1, job_id1)

    # manually re-add parent
    qb.readd_subtask(app1, job_id1)
    validate_one_queued_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
Ejemplo n.º 5
0
def test_rerun_pull_tasks1(app1, app2, job_id1, log, tasks_json_tmpfile):
    # queue and complete app 1. it queues a child
    enqueue(app1, job_id1)
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    validate_zero_queued_task(app1)
    validate_one_queued_task(app2, job_id1)
    # complete app 2
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_zero_queued_task(app2)

    # readd app 2
    qb.readd_subtask(app2, job_id1)
    validate_zero_queued_task(app1)
    validate_one_queued_task(app2, job_id1)
    # run app 2.  the parent was previously completed
    run_code(log, tasks_json_tmpfile, app2)
    validate_one_completed_task(app1, job_id1)  # previously completed
    validate_one_completed_task(app2, job_id1)
Ejemplo n.º 6
0
def test_pull_tasks_with_many_children(app1, app2, app3, app4, job_id1,
                                       log, tasks_json_tmpfile):
    enqueue(app4, job_id1)
    validate_one_queued_task(app4, job_id1)
    validate_zero_queued_task(app1)
    validate_zero_queued_task(app2)
    validate_zero_queued_task(app3)

    run_code(log, tasks_json_tmpfile, app4, '--bash_cmd echo app4helloworld')
    validate_zero_queued_task(app4)
    validate_one_queued_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    validate_one_queued_task(app3, job_id1)

    consume_queue(app1)
    qb.set_state(app1, job_id1, completed=True)
    validate_zero_queued_task(app4)
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)
    validate_one_queued_task(app3, job_id1)

    consume_queue(app2)
    qb.set_state(app2, job_id1, completed=True)
    validate_zero_queued_task(app4)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
    validate_one_queued_task(app3, job_id1)

    consume_queue(app3)
    qb.set_state(app3, job_id1, completed=True)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
    validate_one_completed_task(app3, job_id1)
    validate_one_queued_task(app4, job_id1)

    consume_queue(app4)
    qb.set_state(app4, job_id1, completed=True)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
    validate_one_completed_task(app3, job_id1)
    validate_one_completed_task(app4, job_id1)
Ejemplo n.º 7
0
def test_complex_dependencies_readd(depends_on1, depends_on_job_id1,
                                    log, tasks_json_tmpfile):
    job_id = depends_on_job_id1

    # mark everything completed. ensure only last completed parent queues child
    parents = list(api.topological_sort(api.get_parents(depends_on1, job_id)))
    for parent, pjob_id in parents[:-1]:
        qb.set_state(parent, pjob_id, completed=True)
    validate_zero_queued_task(depends_on1)
    qb.set_state(parents[-1][0], parents[-1][1], completed=True)
    validate_one_queued_task(depends_on1, job_id)
    # --> parents should queue our app

    consume_queue(depends_on1)
    qb.set_state(depends_on1, job_id, completed=True)
    validate_one_completed_task(depends_on1, job_id)

    # ok great - ran through pipeline once.
    log.warn("OK... Now try complex dependency test with a readd")
    # re-complete the very first parent.
    # we assume that this parent is a root task
    parent, pjob_id = parents[0]
    qb.readd_subtask(parent, pjob_id)
    validate_one_queued_task(parent, pjob_id)
    validate_zero_queued_task(depends_on1)
    consume_queue(parent)
    qb.set_state(parent, pjob_id, completed=True)
    validate_one_completed_task(parent, pjob_id)
    # since the parent that re-queues children that may be depends_on1's
    # parents, complete those too!
    for p2, pjob2 in api.get_children(parent, pjob_id, False):
        if p2 == depends_on1:
            continue
        consume_queue(p2)
        qb.set_state(p2, pjob2, completed=True)
    # now, that last parent should have queued our application
    validate_n_queued_task(
        depends_on1, job_id,
        job_id.replace('testID1', 'testID3'))  # this replace is a hack
    run_code(log, tasks_json_tmpfile, depends_on1, '--bash_cmd echo 123')
    run_code(log, tasks_json_tmpfile, depends_on1, '--bash_cmd echo 123')
    validate_one_completed_task(depends_on1, job_id)
Ejemplo n.º 8
0
def test_rerun_pull_tasks_when_manually_queuing_child_and_parent(
        app1, app2, job_id1):
    _test_rerun_tasks_when_manually_queuing_child_and_parent(
        app1, app2, job_id1)

    # complete child first
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    # --> parent still queued
    validate_one_queued_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)

    # then complete parent
    qb.set_state(app1, job_id1, completed=True)
    consume_queue(app1)
    # --> child gets re-queued
    validate_one_completed_task(app1, job_id1)
    validate_one_queued_task(app2, job_id1)

    # complete child second time
    qb.set_state(app2, job_id1, completed=True)
    consume_queue(app2)
    validate_one_completed_task(app1, job_id1)
    validate_one_completed_task(app2, job_id1)
Ejemplo n.º 9
0
def test_maybe_add_subtask_priority_second(app1, job_id1, job_id2):
    api.maybe_add_subtask(app1, job_id1, priority=20)
    api.maybe_add_subtask(app1, job_id2, priority=10)
    nose.tools.assert_equal(consume_queue(app1), job_id2)
    nose.tools.assert_equal(consume_queue(app1), job_id1)
Ejemplo n.º 10
0
def test_maybe_add_subtask_no_priority(app1, job_id1, job_id2):
    api.maybe_add_subtask(app1, job_id1)
    api.maybe_add_subtask(app1, job_id2)
    nose.tools.assert_equal(consume_queue(app1), job_id1)
    nose.tools.assert_equal(consume_queue(app1), job_id2)