def test_retry_failed_task(
app1, job_id1, job_id2, log, tasks_json_tmpfile):
"""
Retry failed tasks up to max num retries and then remove self from queue
Tasks should maintain proper task state throughout.
"""
# create 2 tasks in same queue
enqueue(app1, job_id1)
enqueue(app1, job_id2, validate_queued=False)
nose.tools.assert_equal(2, get_qb_status(app1, job_id1)['app_qsize'])
nose.tools.assert_equal(job_id1, cycle_queue(app1))
# run job_id2 and have it fail
run_code(
log, tasks_json_tmpfile, app1,
extra_opts='--bash_cmd "&& notacommand...fail" ')
# ensure we still have both items in the queue
nose.tools.assert_true(get_qb_status(app1, job_id1)['in_queue'])
nose.tools.assert_true(get_qb_status(app1, job_id2)['in_queue'])
# ensure the failed task is sent to back of the queue
nose.tools.assert_equal(2, get_qb_status(app1, job_id1)['app_qsize'])
nose.tools.assert_equal(job_id1, cycle_queue(app1))
# run and fail n times, where n = max failures
run_code(
log, tasks_json_tmpfile, app1,
extra_opts='--max_retry 1 --bash_cmd "&& notacommand...fail"')
# verify that job_id2 is removed from queue
validate_one_queued_task(app1, job_id1)
# verify that job_id2 state is 'failed' and job_id1 is still pending
validate_one_failed_task(app1, job_id2)
def test_pull_tasks1(app1, app2, job_id1, log, tasks_json_tmpfile):
"""
Parent tasks should be generated and executed before child tasks
(The Bubble Up and then Bubble Down test)
If A --> B, and:
we queue and run B, then we should have 0 completed tasks,
but A should be queued
nothing should change until:
we run A and A becomes completed
we then run B and B becomes completed
"""
enqueue(app2, job_id1)
run_code(log, tasks_json_tmpfile, app2)
validate_one_queued_task(app1, job_id1)
validate_zero_queued_task(app2)
run_code(log, tasks_json_tmpfile, app2)
validate_one_queued_task(app1, job_id1)
validate_zero_queued_task(app2)
run_code(log, tasks_json_tmpfile, app1)
validate_one_completed_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
run_code(log, tasks_json_tmpfile, app2)
validate_one_completed_task(app2, job_id1)
def test_rerun_push_tasks1(app1, app2, job_id1):
# this tests recursively deleteing parent status on child nodes
# queue and complete app 1. it queues a child
enqueue(app1, job_id1)
qb.set_state(app1, job_id1, completed=True)
consume_queue(app1)
validate_one_completed_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
# complete app 2
qb.set_state(app2, job_id1, completed=True)
consume_queue(app2)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
# readd app 1
qb.readd_subtask(app1, job_id1)
validate_one_queued_task(app1, job_id1)
validate_zero_queued_task(app2)
nose.tools.assert_true(
qb.check_state(app2, job_id1, pending=True))
# complete app 1
qb.set_state(app1, job_id1, completed=True)
consume_queue(app1)
validate_one_completed_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
# complete app 2
qb.set_state(app2, job_id1, completed=True)
consume_queue(app2)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
def test_valid_if_or_func2(app3, job_id1, job_id2, job_id3):
"""Verify that the valid_if_or option supports the "_func" option
app_name: {"valid_if_or": {"_func": "python.import.path.to.func"}}
where the function definition looks like: func(**parsed_job_id)
"""
enqueue(app3, job_id3, validate_queued=False)
validate_one_skipped_task(app3, job_id3)
def test_valid_if_or_func3(app3, job_id1, job_id2, job_id3):
"""Verify that the valid_if_or option supports the "_func" option
app_name: {"valid_if_or": {"_func": "python.import.path.to.func"}}
where the function definition looks like: func(**parsed_job_id)
"""
# if the job_id matches the valid_if_or: {"_func": func...} criteria, then:
enqueue(app3, job_id1, validate_queued=False)
validate_one_queued_task(app3, job_id1)
def test_run_given_specific_job_id(app1, job_id1, log, tasks_json_tmpfile):
enqueue(app1, job_id1)
out, err = run_code(
log, tasks_json_tmpfile, app1,
'--job_id %s' % job_id1, raise_on_err=False, capture=True)
nose.tools.assert_regexp_matches(err, (
'UserWarning: Will not execute this task because it might be'
' already queued or completed!'))
validate_one_queued_task(app1, job_id1)
def test_app_has_command_line_params(
bash1, job_id1, log, tasks_json_tmpfile):
enqueue(bash1, job_id1)
msg = 'output: %s'
# Test passed in params exist
_, logoutput = run_code(
log, tasks_json_tmpfile, bash1,
extra_opts='--redirect_to_stderr --bash_cmd echo newfakereadfp',
capture=True, raise_on_err=True)
nose.tools.assert_in(
'newfakereadfp', logoutput, msg % logoutput)
def test_push_tasks1(app1, app2, job_id1, log, tasks_json_tmpfile):
"""
Child tasks should be generated and executed properly
if task A --> task B, and we queue & run A,
then we should end up with one A task completed and one queued B task
"""
enqueue(app1, job_id1)
run_code(log, tasks_json_tmpfile, app1)
validate_one_completed_task(app1, job_id1)
# check child
validate_one_queued_task(app2, job_id1)
def test_bash1(bash1, job_id1, log, tasks_json_tmpfile):
"""a bash task should execute properly """
# queue task
enqueue(bash1, job_id1)
validate_one_queued_task(bash1, job_id1)
# run failing task
run_code(
log, tasks_json_tmpfile, bash1, '--bash_cmd thiscommandshouldfail')
validate_one_queued_task(bash1, job_id1)
# run successful task
run_code(log, tasks_json_tmpfile, bash1, '--bash_cmd echo 123')
validate_zero_queued_task(bash1)
def test_get_qsize(app1, job_id1, job_id2):
nt.assert_equal(api.get_qsize(app1, queued=True, taken=True), 0)
tt.enqueue(app1, job_id1, )
tt.enqueue(app1, job_id2, validate_queued=False)
q = api.get_qbclient().LockingQueue(app1)
itm = q.get()
nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=True))
nt.assert_equal(1, api.get_qsize(app1, queued=False, taken=True))
nt.assert_equal(1, api.get_qsize(app1, queued=True, taken=False))
q.put(itm)
q.consume()
nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=True))
nt.assert_equal(0, api.get_qsize(app1, queued=False, taken=True))
nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=False))
def test_skipped_parent_and_queued_child(app1, app2, app3, app4, job_id1,
log, tasks_json_tmpfile):
qb.set_state(app1, job_id1, skipped=True)
qb.set_state(app3, job_id1, skipped=True)
qb.set_state(app2, job_id1, skipped=True)
enqueue(app4, job_id1)
validate_zero_queued_task(app2)
validate_one_queued_task(app4, job_id1)
# ensure child unqueues itself and raises warning
out, err = run_code(log, tasks_json_tmpfile, app4, capture=True)
nose.tools.assert_in(
"parent_job_id is marked as 'skipped', so should be impossible for me,"
" the child, to exist", err)
validate_zero_queued_task(app4)
validate_zero_queued_task(app2)
def test_complex_dependencies_pull_push(
depends_on1, depends_on_job_id1, log, tasks_json_tmpfile):
job_id = depends_on_job_id1
enqueue(depends_on1, job_id)
run_code(log, tasks_json_tmpfile, depends_on1, '--bash_cmd echo 123')
parents = api.get_parents(depends_on1, job_id)
parents = list(api.topological_sort(parents))
for parent, pjob_id in parents[:-1]:
qb.set_state(parent, pjob_id, completed=True)
validate_zero_queued_task(depends_on1)
qb.set_state(*parents[-1], completed=True)
validate_one_queued_task(depends_on1, job_id)
run_code(log, tasks_json_tmpfile, depends_on1, '--bash_cmd echo 123')
validate_one_completed_task(depends_on1, job_id)
def test_child_running_while_parent_pending_but_not_executing(
app1, app2, job_id1):
enqueue(app1, job_id1)
enqueue(app2, job_id1)
parents_completed, consume_queue, parent_lock = \
qb.ensure_parents_completed(app2, job_id1)
# ensure lock is obtained by ensure_parents_completed
validate_one_queued_executing_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
nose.tools.assert_equal(parents_completed, False)
# child should promise to remove itself from queue
nose.tools.assert_equal(consume_queue, True)
nose.tools.assert_is_instance(parent_lock, qb.BaseLock)
# cleanup
parent_lock.release()
def test_child_running_while_parent_pending_and_executing(
app1, app2, job_id1):
enqueue(app1, job_id1)
enqueue(app2, job_id1)
lock = qb.obtain_execute_lock(app1, job_id1)
assert lock
parents_completed, consume_queue, parent_lock = \
qb.ensure_parents_completed(app2, job_id1)
validate_one_queued_executing_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
nose.tools.assert_equal(parents_completed, False)
# child should not promise to remove itself from queue
nose.tools.assert_equal(consume_queue, False)
nose.tools.assert_is_none(parent_lock)
# cleanup
lock.release()
def test_get_qsize(app1, job_id1, job_id2):
nt.assert_equal(api.get_qsize(app1, queued=True, taken=True), 0)
tt.enqueue(
app1,
job_id1,
)
tt.enqueue(app1, job_id2, validate_queued=False)
q = api.get_qbclient().LockingQueue(app1)
itm = q.get()
nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=True))
nt.assert_equal(1, api.get_qsize(app1, queued=False, taken=True))
nt.assert_equal(1, api.get_qsize(app1, queued=True, taken=False))
q.put(itm)
q.consume()
nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=True))
nt.assert_equal(0, api.get_qsize(app1, queued=False, taken=True))
nt.assert_equal(2, api.get_qsize(app1, queued=True, taken=False))
def test_should_not_add_queue_while_consuming_queue(app1, job_id1):
# This test guards from doubly queuing jobs
# This protects from simultaneous operations on root and leaf nodes
# ie (parent and child) for the following operations:
# adding, readding or a mix of both
enqueue(app1, job_id1)
q = qb.get_qbclient().LockingQueue(app1)
q.get()
validate_one_queued_task(app1, job_id1)
enqueue(app1, job_id1)
with nose.tools.assert_raises(exceptions.JobAlreadyQueued):
qb.readd_subtask(app1, job_id1)
validate_one_queued_task(app1, job_id1)
# cleanup
q.consume()
def test_rerun_pull_tasks1(app1, app2, job_id1, log, tasks_json_tmpfile):
# queue and complete app 1. it queues a child
enqueue(app1, job_id1)
qb.set_state(app1, job_id1, completed=True)
consume_queue(app1)
validate_zero_queued_task(app1)
validate_one_queued_task(app2, job_id1)
# complete app 2
qb.set_state(app2, job_id1, completed=True)
consume_queue(app2)
validate_zero_queued_task(app2)
# readd app 2
qb.readd_subtask(app2, job_id1)
validate_zero_queued_task(app1)
validate_one_queued_task(app2, job_id1)
# run app 2. the parent was previously completed
run_code(log, tasks_json_tmpfile, app2)
validate_one_completed_task(app1, job_id1) # previously completed
validate_one_completed_task(app2, job_id1)
def _test_rerun_tasks_when_manually_queuing_child_and_parent(
app1, app2, job_id1):
# complete parent and child
enqueue(app1, job_id1)
qb.set_state(app1, job_id1, completed=True)
consume_queue(app1)
qb.set_state(app2, job_id1, completed=True)
consume_queue(app2)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
# manually re-add child
qb.readd_subtask(app2, job_id1)
validate_one_queued_task(app2, job_id1)
validate_one_completed_task(app1, job_id1)
# manually re-add parent
qb.readd_subtask(app1, job_id1)
validate_one_queued_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
def test_pull_tasks_with_many_children(app1, app2, app3, app4, job_id1,
log, tasks_json_tmpfile):
enqueue(app4, job_id1)
validate_one_queued_task(app4, job_id1)
validate_zero_queued_task(app1)
validate_zero_queued_task(app2)
validate_zero_queued_task(app3)
run_code(log, tasks_json_tmpfile, app4, '--bash_cmd echo app4helloworld')
validate_zero_queued_task(app4)
validate_one_queued_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
validate_one_queued_task(app3, job_id1)
consume_queue(app1)
qb.set_state(app1, job_id1, completed=True)
validate_zero_queued_task(app4)
validate_one_completed_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
validate_one_queued_task(app3, job_id1)
consume_queue(app2)
qb.set_state(app2, job_id1, completed=True)
validate_zero_queued_task(app4)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
validate_one_queued_task(app3, job_id1)
consume_queue(app3)
qb.set_state(app3, job_id1, completed=True)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
validate_one_completed_task(app3, job_id1)
validate_one_queued_task(app4, job_id1)
consume_queue(app4)
qb.set_state(app4, job_id1, completed=True)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
validate_one_completed_task(app3, job_id1)
validate_one_completed_task(app4, job_id1)
def test_rerun_manual_task1(app1, job_id1):
enqueue(app1, job_id1)
validate_one_queued_task(app1, job_id1)
with nose.tools.assert_raises(exceptions.JobAlreadyQueued):
qb.readd_subtask(app1, job_id1)
def test_valid_if_or_1(app2, job_id1):
"""Invalid tasks should be automatically completed.
This is a valid_if_or test (aka passes_filter )... bad naming sorry!"""
job_id = job_id1.replace('profile', 'content')
enqueue(app2, job_id, validate_queued=False)
validate_one_skipped_task(app2, job_id)
def test_valid_task1(app2, job_id1):
"""Valid tasks should be automatically completed"""
enqueue(app2, job_id1, validate_queued=True)