def get_lock_if_job_is_runnable(app_name, job_id):
"""Return a lock instance or False. If returning False,
the job is not ready to execute.
"""
available = qb.check_state(
app_name, job_id, pending=True, raise_if_not_exists=True)
if not available:
try:
raise RuntimeError(
"I found a job in queue that wasn't"
" in state pending. This might be a code bug. You"
" probably queued 2+ of the same job!")
except RuntimeError as err:
# force a traceback in the logs
log.exception(
err, extra=dict(
app_name=app_name,
job_id=job_id,
state=qb.check_state(
app_name, job_id, _get=True)))
return False
l = qb.obtain_execute_lock(app_name, job_id, blocking=False)
if l is False:
log.warn('Could not obtain execute lock for task because'
' something is already processing this job_id',
extra=dict(app_name=app_name, job_id=job_id))
return False
return l
def get_lock_if_job_is_runnable(app_name, job_id):
"""Return a lock instance or False. If returning False,
the job is not ready to execute.
"""
available = qb.check_state(app_name,
job_id,
pending=True,
raise_if_not_exists=True)
if not available:
try:
raise RuntimeError(
"I found a job in queue that wasn't"
" in state pending. This might be a code bug. You"
" probably queued 2+ of the same job!")
except RuntimeError as err:
# force a traceback in the logs
log.exception(err,
extra=dict(app_name=app_name,
job_id=job_id,
state=qb.check_state(app_name,
job_id,
_get=True)))
return False
l = qb.obtain_execute_lock(app_name, job_id, blocking=False)
if l is False:
log.warn(
'Could not obtain execute lock for task because'
' something is already processing this job_id',
extra=dict(app_name=app_name, job_id=job_id))
return False
return l
def test_race_condition_when_parent_queues_child(
app1, app2, job_id1, log, tasks_json_tmpfile):
# The parent queues the child and the child runs before the parent gets
# a chance to mark itself as completed
qb.set_state(app1, job_id1, pending=True)
lock = qb.obtain_execute_lock(app1, job_id1)
assert lock
qb.set_state(app1, job_id1, completed=True)
qb.set_state(app1, job_id1, pending=True)
validate_one_queued_task(app2, job_id1)
validate_zero_queued_task(app1)
# should not complete child. should de-queue child
# should not queue parent.
# should exit gracefully
# stays in the queue (forever) until parent state is completed
run_code(log, tasks_json_tmpfile, app2)
validate_zero_queued_task(app1)
validate_one_queued_task(app2, job_id1)
qb.set_state(
app1, job_id1, completed=True,
_disable_maybe_queue_children_for_testing_only=True)
lock.release()
validate_one_completed_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
run_code(log, tasks_json_tmpfile, app2)
validate_one_completed_task(app1, job_id1)
validate_one_completed_task(app2, job_id1)
def test_child_running_while_parent_pending_and_executing(
app1, app2, job_id1):
enqueue(app1, job_id1)
enqueue(app2, job_id1)
lock = qb.obtain_execute_lock(app1, job_id1)
assert lock
parents_completed, consume_queue, parent_lock = \
qb.ensure_parents_completed(app2, job_id1)
validate_one_queued_executing_task(app1, job_id1)
validate_one_queued_task(app2, job_id1)
nose.tools.assert_equal(parents_completed, False)
# child should not promise to remove itself from queue
nose.tools.assert_equal(consume_queue, False)
nose.tools.assert_is_none(parent_lock)
# cleanup
lock.release()
def _handle_manually_given_job_id(ns):
"""This process was given a specific --job_id arg.
Decide whether it's okay to execute this job_id,
and if its okay to go forward, set job_id state appropriately
"""
log.warn(
('using specific job_id and'
' blindly assuming this job is not already queued.'),
extra=dict(app_name=ns.app_name, job_id=ns.job_id))
if qb.get_qbclient().exists(qb.get_job_path(ns.app_name, ns.job_id)):
msg = ('Will not execute this task because it might be already'
' queued or completed!')
log.critical(
msg, extra=dict(app_name=ns.app_name, job_id=ns.job_id))
raise UserWarning(msg)
lock = qb.obtain_execute_lock(
ns.app_name, ns.job_id, safe=False, raise_on_error=True,
blocking=False)
qb.set_state(ns.app_name, ns.job_id, pending=True)
return lock
def _handle_manually_given_job_id(ns):
"""This process was given a specific --job_id arg.
Decide whether it's okay to execute this job_id,
and if its okay to go forward, set job_id state appropriately
"""
log.warn(('using specific job_id and'
' blindly assuming this job is not already queued.'),
extra=dict(app_name=ns.app_name, job_id=ns.job_id))
if qb.get_qbclient().exists(qb.get_job_path(ns.app_name, ns.job_id)):
msg = ('Will not execute this task because it might be already'
' queued or completed!')
log.critical(msg, extra=dict(app_name=ns.app_name, job_id=ns.job_id))
raise UserWarning(msg)
lock = qb.obtain_execute_lock(ns.app_name,
ns.job_id,
safe=False,
raise_on_error=True,
blocking=False)
qb.set_state(ns.app_name, ns.job_id, pending=True)
return lock
