def test_rerun_push_tasks1(app1, app2, job_id1):
# this tests recursively deleteing parent status on child nodes
# queue and complete app 1. it queues a child
enqueue(app1, job_id1)
qb.set_state(app1, job_id1, completed=True)
consume_queue(app1)
validate_one_completed_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
# complete app 2
qb.set_state(app2, job_id1, completed=True)
consume_queue(app2)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
# readd app 1
qb.readd_subtask(app1, job_id1)
validate_one_queued_task(app1, job_id1)
validate_zero_queued_task(app2)
nose.tools.assert_true(
qb.check_state(app2, job_id1, pending=True))
# complete app 1
qb.set_state(app1, job_id1, completed=True)
consume_queue(app1)
validate_one_completed_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
# complete app 2
qb.set_state(app2, job_id1, completed=True)
consume_queue(app2)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
def _send_to_back_of_queue(q, app_name, job_id):
# this exists so un-runnable tasks don't hog the front of the queue
# and soak up resources
try:
qb.readd_subtask(app_name, job_id, _force=True)
q.consume()
log.info("Job sent to back of queue",
extra=dict(app_name=app_name, job_id=job_id))
except exceptions.JobAlreadyQueued:
log.info("Job already queued. Cannot send to back of queue.",
extra=dict(app_name=app_name, job_id=job_id))
def _send_to_back_of_queue(q, app_name, job_id):
# this exists so un-runnable tasks don't hog the front of the queue
# and soak up resources
try:
qb.readd_subtask(app_name, job_id, _force=True)
q.consume()
log.info(
"Job sent to back of queue",
extra=dict(app_name=app_name, job_id=job_id))
except exceptions.JobAlreadyQueued:
log.info(
"Job already queued. Cannot send to back of queue.",
extra=dict(app_name=app_name, job_id=job_id))
def test_should_not_add_queue_while_consuming_queue(app1, job_id1):
# This test guards from doubly queuing jobs
# This protects from simultaneous operations on root and leaf nodes
# ie (parent and child) for the following operations:
# adding, readding or a mix of both
enqueue(app1, job_id1)
q = qb.get_qbclient().LockingQueue(app1)
q.get()
validate_one_queued_task(app1, job_id1)
enqueue(app1, job_id1)
with nose.tools.assert_raises(exceptions.JobAlreadyQueued):
qb.readd_subtask(app1, job_id1)
validate_one_queued_task(app1, job_id1)
# cleanup
q.consume()
def test_complex_dependencies_readd(depends_on1, depends_on_job_id1,
log, tasks_json_tmpfile):
job_id = depends_on_job_id1
# mark everything completed. ensure only last completed parent queues child
parents = list(api.topological_sort(api.get_parents(depends_on1, job_id)))
for parent, pjob_id in parents[:-1]:
qb.set_state(parent, pjob_id, completed=True)
validate_zero_queued_task(depends_on1)
qb.set_state(parents[-1][0], parents[-1][1], completed=True)
validate_one_queued_task(depends_on1, job_id)
# --> parents should queue our app
consume_queue(depends_on1)
qb.set_state(depends_on1, job_id, completed=True)
validate_one_completed_task(depends_on1, job_id)
# ok great - ran through pipeline once.
log.warn("OK... Now try complex dependency test with a readd")
# re-complete the very first parent.
# we assume that this parent is a root task
parent, pjob_id = parents[0]
qb.readd_subtask(parent, pjob_id)
validate_one_queued_task(parent, pjob_id)
validate_zero_queued_task(depends_on1)
consume_queue(parent)
qb.set_state(parent, pjob_id, completed=True)
validate_one_completed_task(parent, pjob_id)
# since the parent that re-queues children that may be depends_on1's
# parents, complete those too!
for p2, pjob2 in api.get_children(parent, pjob_id, False):
if p2 == depends_on1:
continue
consume_queue(p2)
qb.set_state(p2, pjob2, completed=True)
# now, that last parent should have queued our application
validate_n_queued_task(
depends_on1, job_id,
job_id.replace('testID1', 'testID3')) # this replace is a hack
run_code(log, tasks_json_tmpfile, depends_on1, '--bash_cmd echo 123')
run_code(log, tasks_json_tmpfile, depends_on1, '--bash_cmd echo 123')
validate_one_completed_task(depends_on1, job_id)
def _test_rerun_tasks_when_manually_queuing_child_and_parent(
app1, app2, job_id1):
# complete parent and child
enqueue(app1, job_id1)
qb.set_state(app1, job_id1, completed=True)
consume_queue(app1)
qb.set_state(app2, job_id1, completed=True)
consume_queue(app2)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
# manually re-add child
qb.readd_subtask(app2, job_id1)
validate_one_queued_task(app2, job_id1)
validate_one_completed_task(app1, job_id1)
# manually re-add parent
qb.readd_subtask(app1, job_id1)
validate_one_queued_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
def test_rerun_pull_tasks1(app1, app2, job_id1, log, tasks_json_tmpfile):
# queue and complete app 1. it queues a child
enqueue(app1, job_id1)
qb.set_state(app1, job_id1, completed=True)
consume_queue(app1)
validate_zero_queued_task(app1)
validate_one_queued_task(app2, job_id1)
# complete app 2
qb.set_state(app2, job_id1, completed=True)
consume_queue(app2)
validate_zero_queued_task(app2)
# readd app 2
qb.readd_subtask(app2, job_id1)
validate_zero_queued_task(app1)
validate_one_queued_task(app2, job_id1)
# run app 2. the parent was previously completed
run_code(log, tasks_json_tmpfile, app2)
validate_one_completed_task(app1, job_id1) # previously completed
validate_one_completed_task(app2, job_id1)
def test_rerun_manual_task2(app1, job_id1):
qb.readd_subtask(app1, job_id1)
validate_one_queued_task(app1, job_id1)
def test_rerun_manual_task1(app1, job_id1):
enqueue(app1, job_id1)
validate_one_queued_task(app1, job_id1)
with nose.tools.assert_raises(exceptions.JobAlreadyQueued):
qb.readd_subtask(app1, job_id1)