def get_lock_if_job_is_runnable(app_name, job_id): """Return a lock instance or False. If returning False, the job is not ready to execute. """ available = qb.check_state( app_name, job_id, pending=True, raise_if_not_exists=True) if not available: try: raise RuntimeError( "I found a job in queue that wasn't" " in state pending. This might be a code bug. You" " probably queued 2+ of the same job!") except RuntimeError as err: # force a traceback in the logs log.exception( err, extra=dict( app_name=app_name, job_id=job_id, state=qb.check_state( app_name, job_id, _get=True))) return False l = qb.obtain_execute_lock(app_name, job_id, blocking=False) if l is False: log.warn('Could not obtain execute lock for task because' ' something is already processing this job_id', extra=dict(app_name=app_name, job_id=job_id)) return False return l
def get_lock_if_job_is_runnable(app_name, job_id): """Return a lock instance or False. If returning False, the job is not ready to execute. """ available = qb.check_state(app_name, job_id, pending=True, raise_if_not_exists=True) if not available: try: raise RuntimeError( "I found a job in queue that wasn't" " in state pending. This might be a code bug. You" " probably queued 2+ of the same job!") except RuntimeError as err: # force a traceback in the logs log.exception(err, extra=dict(app_name=app_name, job_id=job_id, state=qb.check_state(app_name, job_id, _get=True))) return False l = qb.obtain_execute_lock(app_name, job_id, blocking=False) if l is False: log.warn( 'Could not obtain execute lock for task because' ' something is already processing this job_id', extra=dict(app_name=app_name, job_id=job_id)) return False return l
def test_race_condition_when_parent_queues_child( app1, app2, job_id1, log, tasks_json_tmpfile): # The parent queues the child and the child runs before the parent gets # a chance to mark itself as completed qb.set_state(app1, job_id1, pending=True) lock = qb.obtain_execute_lock(app1, job_id1) assert lock qb.set_state(app1, job_id1, completed=True) qb.set_state(app1, job_id1, pending=True) validate_one_queued_task(app2, job_id1) validate_zero_queued_task(app1) # should not complete child. should de-queue child # should not queue parent. # should exit gracefully # stays in the queue (forever) until parent state is completed run_code(log, tasks_json_tmpfile, app2) validate_zero_queued_task(app1) validate_one_queued_task(app2, job_id1) qb.set_state( app1, job_id1, completed=True, _disable_maybe_queue_children_for_testing_only=True) lock.release() validate_one_completed_task(app1, job_id1) validate_one_queued_task(app2, job_id1) run_code(log, tasks_json_tmpfile, app2) validate_one_completed_task(app1, job_id1) validate_one_completed_task(app2, job_id1)
def test_child_running_while_parent_pending_and_executing( app1, app2, job_id1): enqueue(app1, job_id1) enqueue(app2, job_id1) lock = qb.obtain_execute_lock(app1, job_id1) assert lock parents_completed, consume_queue, parent_lock = \ qb.ensure_parents_completed(app2, job_id1) validate_one_queued_executing_task(app1, job_id1) validate_one_queued_task(app2, job_id1) nose.tools.assert_equal(parents_completed, False) # child should not promise to remove itself from queue nose.tools.assert_equal(consume_queue, False) nose.tools.assert_is_none(parent_lock) # cleanup lock.release()
def _handle_manually_given_job_id(ns): """This process was given a specific --job_id arg. Decide whether it's okay to execute this job_id, and if its okay to go forward, set job_id state appropriately """ log.warn( ('using specific job_id and' ' blindly assuming this job is not already queued.'), extra=dict(app_name=ns.app_name, job_id=ns.job_id)) if qb.get_qbclient().exists(qb.get_job_path(ns.app_name, ns.job_id)): msg = ('Will not execute this task because it might be already' ' queued or completed!') log.critical( msg, extra=dict(app_name=ns.app_name, job_id=ns.job_id)) raise UserWarning(msg) lock = qb.obtain_execute_lock( ns.app_name, ns.job_id, safe=False, raise_on_error=True, blocking=False) qb.set_state(ns.app_name, ns.job_id, pending=True) return lock
def _handle_manually_given_job_id(ns): """This process was given a specific --job_id arg. Decide whether it's okay to execute this job_id, and if its okay to go forward, set job_id state appropriately """ log.warn(('using specific job_id and' ' blindly assuming this job is not already queued.'), extra=dict(app_name=ns.app_name, job_id=ns.job_id)) if qb.get_qbclient().exists(qb.get_job_path(ns.app_name, ns.job_id)): msg = ('Will not execute this task because it might be already' ' queued or completed!') log.critical(msg, extra=dict(app_name=ns.app_name, job_id=ns.job_id)) raise UserWarning(msg) lock = qb.obtain_execute_lock(ns.app_name, ns.job_id, safe=False, raise_on_error=True, blocking=False) qb.set_state(ns.app_name, ns.job_id, pending=True) return lock