def test_failed_job_max_tries_1__move_to_dlq():
q = Queue()
failed_q = get_failed_queue()
dlq = Queue('dead_letter_queue')
# we could test with one worker here, but we don't want
# the test to depend on when the Worker performs maint.
# tasks (before or after processing jobs)
w = Worker([q])
rw = RetryWorker([q], retry_config=dict(max_tries=1, delays=[]))
# run job that will fail
job = q.enqueue(error_fun)
w.work(burst=True)
assert q.count == 0
assert get_failed_queue().count == 1
# run retry worker
rw.work(burst=True)
job.refresh()
assert q.count == 0
assert failed_q.count == 0
assert dlq.count == 1
assert job.meta['tries'] == 1
def overview(queue_name, page):
if queue_name == 'failed':
queue = get_failed_queue()
elif queue_name is None:
# Show the failed queue by default if it contains any jobs
failed = get_failed_queue()
if not failed.is_empty():
queue = failed
else:
queue = Queue()
else:
queue = Queue(queue_name)
r = make_response(
render_template(
'rq_dashboard/dashboard.html',
workers=Worker.all(),
queue=queue,
page=page,
queues=get_all_queues(),
rq_url_prefix=url_for('.overview'),
rq_dashboard_version=rq_dashboard_version,
rq_version=rq_version,
))
r.headers.set('Cache-Control', 'no-store')
return r
def test_failed_job_max_tries_1__move_to_dlq():
q = Queue()
failed_q = get_failed_queue()
dlq = Queue('dead_letter_queue')
# we could test with one worker here, but we don't want
# the test to depend on when the Worker performs maint.
# tasks (before or after processing jobs)
w = Worker([q])
rw = RetryWorker([q], retry_config=dict(
max_tries=1, delays=[]))
# run job that will fail
job = q.enqueue(error_fun)
w.work(burst=True)
assert q.count == 0
assert get_failed_queue().count == 1
# run retry worker
rw.work(burst=True)
job.refresh()
assert q.count == 0
assert failed_q.count == 0
assert dlq.count == 1
assert job.meta['tries'] == 1
def overview(queue_name, page, state=None):
if queue_name == 'failed':
queue = get_failed_queue()
elif queue_name is None:
# Show the failed queue by default if it contains any jobs
failed = get_failed_queue()
if not failed.is_empty():
queue = failed
else:
queue = Queue()
else:
queue = Queue(queue_name)
r = make_response(
render_template('rq_dashboard/dashboard.html',
workers=Worker.all(),
queue=queue,
page=page,
state=state,
queues=get_all_queues(),
rq_url_prefix=url_for('.overview'),
newest_top=current_app.config.get(
'RQ_DASHBOARD_JOB_SORT_ORDER') == '-age'))
r.headers.set('Cache-Control', 'no-store')
return r
def test_requeue_nonfailed_job_fails(self):
"""Requeueing non-failed jobs raises error."""
q = Queue()
job = q.enqueue(say_hello, 'Nick', foo='bar')
# Assert that we cannot requeue a job that's not on the failed queue
with self.assertRaises(InvalidJobOperationError):
get_failed_queue().requeue(job.id)
def test_requeue_nonfailed_job_fails(self):
"""Requeueing non-failed jobs raises error."""
q = Queue()
job = q.enqueue(say_hello, 'Nick', foo='bar')
# Assert that we cannot requeue a job that's not on the failed queue
with self.assertRaises(InvalidJobOperationError):
get_failed_queue().requeue(job.id)
def test_requeue_sets_status_to_queued(self):
"""Requeueing a job should set its status back to QUEUED."""
job = Job.create(func=div_by_zero, args=(1, 2, 3))
job.save()
get_failed_queue().quarantine(job, Exception('Some fake error'))
get_failed_queue().requeue(job.id)
job = Job.fetch(job.id)
self.assertEqual(job.status, Status.QUEUED)
def test_quarantine_preserves_timeout(self):
"""Quarantine preserves job timeout."""
job = Job.create(func=div_by_zero, args=(1, 2, 3))
job.origin = 'fake'
job.timeout = 200
job.save()
get_failed_queue().quarantine(job, Exception('Some fake error'))
self.assertEquals(job.timeout, 200)
def test_quarantine_preserves_timeout(self):
"""Quarantine preserves job timeout."""
job = Job.create(func=div_by_zero, args=(1, 2, 3))
job.origin = 'fake'
job.timeout = 200
job.save()
get_failed_queue().quarantine(job, Exception('Some fake error'))
self.assertEquals(job.timeout, 200)
def test_requeue_sets_status_to_queued(self):
"""Requeueing a job should set its status back to QUEUED."""
job = Job.create(func=div_by_zero, args=(1, 2, 3))
job.save()
get_failed_queue().quarantine(job, Exception('Some fake error'))
get_failed_queue().requeue(job.id)
job = Job.fetch(job.id)
self.assertEqual(job.get_status(), JobStatus.QUEUED)
def setUp(self):
super(TestRQCli, self).setUp()
db_num = self.testconn.connection_pool.connection_kwargs['db']
self.redis_url = 'redis://127.0.0.1:6379/%d' % db_num
job = Job.create(func=div_by_zero, args=(1, 2, 3))
job.origin = 'fake'
job.save()
get_failed_queue().quarantine(job, Exception('Some fake error')) # noqa
def test_get_failed_queue(self):
"""Use custom job class"""
class CustomJob(Job):
pass
failed_queue = get_failed_queue(job_class=CustomJob)
self.assertIs(failed_queue.job_class, CustomJob)
failed_queue = get_failed_queue(job_class='rq.job.Job')
self.assertIsNot(failed_queue.job_class, CustomJob)
def test_requeueing_preserves_timeout(self):
"""Requeueing preserves job timeout."""
job = Job.create(func=div_by_zero, args=(1, 2, 3))
job.origin = "fake"
job.timeout = 200
job.save()
get_failed_queue().quarantine(job, Exception("Some fake error"))
get_failed_queue().requeue(job.id)
job = Job.fetch(job.id)
self.assertEquals(job.timeout, 200)
def test_failed_job_max_tries_2__retry_once_then_move_to_dlq():
q = Queue()
q2 = Queue('not_used')
failed_q = get_failed_queue()
dlq = Queue('dead_letter_queue')
w = Worker([q])
# Here the RetryWorker not listening on an active queue: it will not
# run any jobs, just look to requeue failed jobs.
rw = RetryWorker([q2],
retry_config=dict(max_tries=2,
maint_interval=0,
delays=[]))
# run job that will fail
job = q.enqueue(error_fun)
w.work(burst=True)
assert q.count == 0
assert get_failed_queue().count == 1
# run retry worker
rw.work(burst=True)
# job should be requeued
assert q.count == 1
assert failed_q.count == 0
assert dlq.count == 0
job.refresh()
assert job.meta['tries'] == 2
# regular worker runs the job again
w.work(burst=True)
# job fails again
assert q.count == 0
assert failed_q.count == 1
assert dlq.count == 0
job.refresh()
assert job.meta['tries'] == 2
# run retry worker
rw.work(burst=True)
# job should be in dlq
assert q.count == 0
assert failed_q.count == 0
assert dlq.count == 1
job.refresh()
assert job.meta['tries'] == 2
def test_failed_job_max_tries_2__retry_once_then_move_to_dlq():
q = Queue()
q2 = Queue('not_used')
failed_q = get_failed_queue()
dlq = Queue('dead_letter_queue')
w = Worker([q])
# Here the RetryWorker not listening on an active queue: it will not
# run any jobs, just look to requeue failed jobs.
rw = RetryWorker([q2], retry_config=dict(
max_tries=2, maint_interval=0, delays=[]))
# run job that will fail
job = q.enqueue(error_fun)
w.work(burst=True)
assert q.count == 0
assert get_failed_queue().count == 1
# run retry worker
rw.work(burst=True)
# job should be requeued
assert q.count == 1
assert failed_q.count == 0
assert dlq.count == 0
job.refresh()
assert job.meta['tries'] == 2
# regular worker runs the job again
w.work(burst=True)
# job fails again
assert q.count == 0
assert failed_q.count == 1
assert dlq.count == 0
job.refresh()
assert job.meta['tries'] == 2
# run retry worker
rw.work(burst=True)
# job should be in dlq
assert q.count == 0
assert failed_q.count == 0
assert dlq.count == 1
job.refresh()
assert job.meta['tries'] == 2
def requeue_all(self):
fq = get_failed_queue()
job_ids = fq.job_ids
count = len(job_ids)
for job_id in job_ids:
requeue_job(job_id)
redirect(URL())
def reschedule_all_failed(request):
queue = get_failed_queue(django_rq.get_connection())
for job in queue.jobs:
requeue_job(job.id, connection=queue.connection)
return HttpResponse('Success')
def test_does_regular_work_like_any_good_worker():
q = Queue()
w = RetryWorker([q])
q.enqueue(success_fun)
w.work(burst=True)
assert q.count == 0
assert get_failed_queue().count == 0
def list_regexp(options):
r = redis.StrictRedis.from_url(REDIS_URL_RQ)
if options.source:
fq = Queue(options.source, connection=r)
else:
fq = get_failed_queue(connection=r)
def exception_matches(regexp, job):
exc_info = '' if job.exc_info is None else job.exc_info
reason = exc_info.split('\n')[:-1]
for r in reason:
match = re.search(regexp, r)
if match:
return True
return False
jobs = fq.get_jobs()
for regexp in options.regexp:
jobs = [job for job in jobs if exception_matches(regexp, job)]
for regexp in options.descr_regexp:
jobs = [job for job in jobs if re.search(regexp, job.description)]
for job in jobs:
print job.id
def list_timeouts(options):
r = redis.StrictRedis.from_url(REDIS_URL_RQ)
fq = get_failed_queue(connection=r)
def get_timeout(job):
exc_info = '' if job.exc_info is None else job.exc_info
reason = exc_info.split('\n')[-2:-1]
for r in reason:
match = re.match('JobTimeoutException.*?(\d+)', r)
if match:
return int(match.group(1))
return None
jobs = fq.get_jobs()
timeouts = map(get_timeout, jobs)
timeouted_jobs = [(job, timeout) for (job, timeout) in zip(jobs, timeouts)
if timeout is not None]
if options.list_nones:
none_jobs = [job for job in fq.get_jobs() if job.exc_info is None]
for job in none_jobs:
print job.id
else:
for job, to in timeouted_jobs:
print "%s\t\ttimeouted at: %d" % (job.id, to)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
setup_redis(args)
try:
if args.empty_failed_queue:
num_jobs = get_failed_queue().empty()
print('{} jobs removed from failed queue'.format(num_jobs))
else:
if args.only_queues:
func = show_queues
elif args.only_workers:
func = show_workers
else:
func = show_both
interval(args.interval, func, args)
except ConnectionError as e:
print(e)
sys.exit(1)
except KeyboardInterrupt:
print()
sys.exit(0)
def list_messages():
if request.method == 'POST':
if request.form['action'] == 'resend':
comment_id = request.form['comment_id']
comment_submitted(sender=None,
comment=Comment.query.get(comment_id))
elif request.form['action'] == 'rerun':
job_id = request.form['job_id']
redis_url = current_app.config['REDIS_URL']
with Connection(redis.from_url(redis_url)):
fq = get_failed_queue()
fq.requeue(job_id)
else:
page = request.args.get('page', 1, type=int)
pagination = Message.query.paginate(
page,
per_page=get_setting('items_per_page').value,
error_out=False)
messages = pagination.items
with Connection(redis.from_url(current_app.config['REDIS_URL'])):
queue = Queue()
return current_plugin.render_template('list.html',
messages=messages,
queue=queue,
pagination={
'pagination': pagination,
'fragment': {},
'url_for': plugin_url_for,
'url_for_params': {
'args': ['list'],
'kwargs': {
'_component': 'admin'
}
}
})
def delete(self, id_):
'''
Delete a failed task with the given id_.
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:query id: the job ID to delete
:>header Content-Type: application/json
:status 200: ok
:status 401: authentication required
:status 403: you must be an administrator
:status 404: no job exists with this ID
'''
with rq.Connection(g.redis):
found = False
for job in rq.get_failed_queue().jobs:
if job.id == id_:
job.delete()
found = True
break
if not found:
raise NotFound('No job exists with ID "%s".' % id_)
return jsonify(message='ok')
def requeue_all():
fq = get_failed_queue()
job_ids = fq.job_ids
count = len(job_ids)
for job_id in job_ids:
requeue_job(job_id, connection=current_app.redis_conn)
return dict(status='OK', count=count)
def test_does_regular_work_like_any_good_worker():
q = Queue()
w = RetryWorker([q])
q.enqueue(success_fun)
w.work(burst=True)
assert q.count == 0
assert get_failed_queue().count == 0
def test_run_access_self(self):
"""Schedule a job, then run the worker as subprocess"""
q = Queue()
q.enqueue(access_self)
subprocess.check_call(['rqworker', '-u', self.redis_url, '-b'])
assert get_failed_queue().count == 0
assert q.count == 0
def delete(self, id_):
'''
Delete a failed task with the given id_.
:<header Content-Type: application/json
:<header X-Auth: the client's auth token
:query id: the job ID to delete
:>header Content-Type: application/json
:status 200: ok
:status 401: authentication required
:status 403: you must be an administrator
:status 404: no job exists with this ID
'''
with rq.Connection(g.redis):
found = False
for job in rq.get_failed_queue().jobs:
if job.id == id_:
job.delete()
found = True
break
if not found:
raise NotFound('No job exists with ID "%s".' % id_)
return jsonify(message='ok')
def set_up(cls, config):
for queue_name in ("default", "email", "kpi"):
queue = Queue(connection=Redis(
host=config["host"],
port=int(config["port"]),
password=config["password"],
),
**config["queues"][queue_name])
setattr(cls, queue_name, queue)
cls.scheduler = Scheduler(connection=Redis(
host=config["host"],
port=int(config["port"]),
password=config["password"],
))
cls.failed_queue = get_failed_queue(connection=Redis(
host=config["host"],
port=int(config["port"]),
password=config["password"],
))
push_connection(
Redis(
host=config["host"],
port=int(config["port"]),
password=config["password"],
))
def test_self_modification_persistence_with_error(self):
"""Make sure that any meta modification done by
the job itself persists completely through the
queue/worker/job stack -- even if the job errored"""
q = Queue()
failed_q = get_failed_queue()
# Also make sure that previously existing metadata
# persists properly
job = q.enqueue(modify_self_and_error, meta={'foo': 'bar', 'baz': 42},
args=[{'baz': 10, 'newinfo': 'waka'}])
w = Worker([q])
w.work(burst=True)
# Postconditions
self.assertEqual(q.count, 0)
self.assertEqual(failed_q.count, 1)
self.assertEqual(w.get_current_job_id(), None)
job_check = Job.fetch(job.id)
self.assertEqual(set(job_check.meta.keys()),
set(['foo', 'baz', 'newinfo']))
self.assertEqual(job_check.meta['foo'], 'bar')
self.assertEqual(job_check.meta['baz'], 10)
self.assertEqual(job_check.meta['newinfo'], 'waka')
def test_work_fails(self):
"""Failing jobs are put on the failed queue."""
q = Queue()
failed_q = get_failed_queue()
# Preconditions
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# Action
job = q.enqueue(div_by_zero)
self.assertEquals(q.count, 1)
# keep for later
enqueued_at_date = strip_microseconds(job.enqueued_at)
w = Worker([q])
w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 1)
# Check the job
job = Job.fetch(job.id)
self.assertEquals(job.origin, q.name)
# Should be the original enqueued_at date, not the date of enqueueing
# to the failed queue
self.assertEquals(job.enqueued_at, enqueued_at_date)
self.assertIsNotNone(job.exc_info) # should contain exc_info
def test_self_modification_persistence_with_error(self):
"""Make sure that any meta modification done by
the job itself persists completely through the
queue/worker/job stack -- even if the job errored"""
q = Queue()
failed_q = get_failed_queue()
# Also make sure that previously existing metadata
# persists properly
job = q.enqueue(modify_self_and_error,
meta={
'foo': 'bar',
'baz': 42
},
args=[{
'baz': 10,
'newinfo': 'waka'
}])
w = Worker([q])
w.work(burst=True)
# Postconditions
self.assertEqual(q.count, 0)
self.assertEqual(failed_q.count, 1)
self.assertEqual(w.get_current_job_id(), None)
job_check = Job.fetch(job.id)
self.assertEqual(set(job_check.meta.keys()),
set(['foo', 'baz', 'newinfo']))
self.assertEqual(job_check.meta['foo'], 'bar')
self.assertEqual(job_check.meta['baz'], 10)
self.assertEqual(job_check.meta['newinfo'], 'waka')
def test_work_is_unreadable(self):
"""Unreadable jobs are put on the failed queue."""
q = Queue()
failed_q = get_failed_queue()
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# NOTE: We have to fake this enqueueing for this test case.
# What we're simulating here is a call to a function that is not
# importable from the worker process.
job = Job.create(func=div_by_zero, args=(3, ))
job.save()
data = self.testconn.hget(job.key, 'data')
invalid_data = data.replace(b'div_by_zero', b'nonexisting')
assert data != invalid_data
self.testconn.hset(job.key, 'data', invalid_data)
# We use the low-level internal function to enqueue any data (bypassing
# validity checks)
q.push_job_id(job.id)
self.assertEquals(q.count, 1)
# All set, we're going to process it
w = Worker([q])
w.work(burst=True) # should silently pass
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 1)
def test_run_scheduled_access_self(self):
"""Schedule a job that schedules a job, then run the worker as subprocess"""
q = Queue()
q.enqueue(schedule_access_self)
subprocess.check_call(['rqworker', '-u', self.redis_url, '-b'])
assert get_failed_queue().count == 0
assert q.count == 0
def test_custom_exc_handling(self):
"""Custom exception handling."""
def black_hole(job, *exc_info):
# Don't fall through to default behaviour (moving to failed queue)
return False
q = Queue()
failed_q = get_failed_queue()
# Preconditions
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# Action
job = q.enqueue(div_by_zero)
self.assertEquals(q.count, 1)
w = Worker([q], exc_handler=black_hole)
w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 0)
# Check the job
job = Job.fetch(job.id)
self.assertEquals(job.is_failed, True)
def requeue_all():
fq = get_failed_queue()
job_ids = fq.job_ids
count = len(job_ids)
for job_id in job_ids:
requeue_job(job_id)
return dict(status='OK', count=count)
def test_work_fails(self):
"""Failing jobs are put on the failed queue."""
q = Queue()
failed_q = get_failed_queue()
# Preconditions
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# Action
job = q.enqueue(div_by_zero)
self.assertEquals(q.count, 1)
# keep for later
enqueued_at_date = strip_microseconds(job.enqueued_at)
w = Worker([q])
w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 1)
# Check the job
job = Job.fetch(job.id)
self.assertEquals(job.origin, q.name)
# Should be the original enqueued_at date, not the date of enqueueing
# to the failed queue
self.assertEquals(job.enqueued_at, enqueued_at_date)
self.assertIsNotNone(job.exc_info) # should contain exc_info
def requeue_2xtimeout_job_view(job_id):
# Get the handle for the failed queue
fq = get_failed_queue()
# Fetch the job from the failed queue
job = fq.fetch_job(job_id)
# Test if the job exists
if job is None:
raise NoSuchJobError(
'Job {} does not exist in failed queue'.format(job_id)
)
# Remove the job from the failed queue
if fq.remove(job_id) == 0:
raise InvalidJobOperationError('Cannot requeue non-failed jobs')
# Reset the job state
job.set_status(JobStatus.QUEUED)
job.exc_info = None
if not job.timeout:
job.timeout = Queue.DEFAULT_TIMEOUT
# Double the timeout
job.timeout *= 2
# Get a handle for the original queue
q = Queue(job.origin, connection=job.connection)
# Queue the job
q.enqueue_job(job)
return dict(status='OK')
def requeue_all():
fq = get_failed_queue()
job_ids = fq.job_ids
count = len(job_ids)
for job_id in job_ids:
requeue_job(job_id, connection=current_app.redis_conn)
return dict(status='OK', count=count)
def test_custom_exc_handling(self):
"""Custom exception handling."""
def black_hole(job, *exc_info):
# Don't fall through to default behaviour (moving to failed queue)
return False
q = Queue()
failed_q = get_failed_queue()
# Preconditions
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# Action
job = q.enqueue(div_by_zero)
self.assertEquals(q.count, 1)
w = Worker([q], exc_handler=black_hole)
w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 0)
# Check the job
job = Job.fetch(job.id)
self.assertEquals(job.is_failed, True)
def get_queue(queue_name):
if queue_name == 'failed':
return get_failed_queue()
elif queue_name == 'scheduled_jobs':
return SchedulerQueue()
else:
return Queue(queue_name)
def test_work_is_unreadable(self):
"""Unreadable jobs are put on the failed queue."""
q = Queue()
failed_q = get_failed_queue()
self.assertEquals(failed_q.count, 0)
self.assertEquals(q.count, 0)
# NOTE: We have to fake this enqueueing for this test case.
# What we're simulating here is a call to a function that is not
# importable from the worker process.
job = Job.create(func=div_by_zero, args=(3,))
job.save()
data = self.testconn.hget(job.key, 'data')
invalid_data = data.replace(b'div_by_zero', b'nonexisting')
assert data != invalid_data
self.testconn.hset(job.key, 'data', invalid_data)
# We use the low-level internal function to enqueue any data (bypassing
# validity checks)
q.push_job_id(job.id)
self.assertEquals(q.count, 1)
# All set, we're going to process it
w = Worker([q])
w.work(burst=True) # should silently pass
self.assertEquals(q.count, 0)
self.assertEquals(failed_q.count, 1)
def requeue(cli_config, all, job_class, job_ids, **options):
"""Requeue failed jobs."""
failed_queue = get_failed_queue(connection=cli_config.connection,
job_class=cli_config.job_class)
if all:
job_ids = failed_queue.job_ids
if not job_ids:
click.echo('Nothing to do')
sys.exit(0)
click.echo('Requeueing {0} jobs from failed queue'.format(len(job_ids)))
fail_count = 0
with click.progressbar(job_ids) as job_ids:
for job_id in job_ids:
try:
failed_queue.requeue(job_id)
except InvalidJobOperationError:
fail_count += 1
if fail_count > 0:
click.secho(
'Unable to requeue {0} jobs from failed queue'.format(fail_count),
fg='red')
def get_operation_status():
result = {}
for op in OPERATIONS:
q = Queue(op, connection=redis_connection())
result[op] = [job_to_dict(j) for j in q.jobs]
fq = get_failed_queue(redis_connection())
result['failed'] = [job_to_dict(j) for j in fq.jobs]
return make_response(body=result)
def __init__(self):
self.redis_url = os.getenv('REDISTOGO_URL', 'redis:#localhost:6379')
self.conn = redis.from_url(self.redis_url)
self.queue = Queue(connection=self.conn)
self.listen = ['default']
self.failed_queue = get_failed_queue(connection=self.conn)
self.scheduler = BackgroundScheduler()
return None
def failed_jobs_by_queue(queue):
failed = []
fq = get_failed_queue(redisClient)
for id in fq.get_job_ids():
try:
if Job.fetch(str(id), connection=redisClient).origin == queue.name:
failed.append(str(id))
except:
continue
return failed
def test_run_scheduled_access_self(self):
"""Schedule a job that schedules a job, then run the worker as subprocess"""
if 'pypy' in sys.version.lower():
# horrible bodge until we drop 2.6 support and can use skipIf
return
q = Queue()
q.enqueue(schedule_access_self)
subprocess.check_call(['rqworker', '-u', self.redis_url, '-b'])
assert get_failed_queue().count == 0
assert q.count == 0
def list_queues(self):
with Connection(self.redis_conn):
l=[]
if len(get_failed_queue().jobs) > 0:
l.append({
'worker': "N/A",
'name': 'failed',
'jobs': len(get_failed_queue().jobs),
'link': URL("admin", "jobs", args=["failed"])})
for w in Worker.all():
for q in w.queues:
l.append({
'worker': w.name,
'name': q.name,
'jobs': q.count,
'link': URL("admin", "jobs", args=[q.name])})
return l
def test_run_scheduled_access_self(self):
"""Schedule a job that schedules a job, then run the worker as subprocess"""
if 'pypy' in sys.version.lower():
# horrible bodge until we drop 2.6 support and can use skipIf
return
q = Queue()
q.enqueue(schedule_access_self)
subprocess.check_call(['rqworker', '-u', self.redis_url, '-b'])
assert get_failed_queue().count == 0
assert q.count == 0
def check_failed_queue(self, redis_conn):
"""
Requeue all jobs in the Failed job queue
"""
with Connection(redis_conn):
failed_queue = get_failed_queue()
# TODO: Need to keep track of number of attempts a job is requeued
for job in failed_queue.get_jobs():
self.logger.info("Requeued: " + str(job.id))
failed_queue.requeue(job.id)
def reschedule_all_failed(request):
from rq import requeue_job
from rq import get_failed_queue
from django_rq import get_connection
queue = get_failed_queue(get_connection())
for job in queue.jobs:
requeue_job(job.id, connection=queue.connection)
return HttpResponse('Success')
def overview(queue_name, page):
if queue_name == 'failed':
queue = get_failed_queue()
elif queue_name is None:
# Show the failed queue by default if it contains any jobs
failed = get_failed_queue()
if not failed.is_empty():
queue = failed
else:
queue = Queue()
else:
queue = Queue(queue_name)
r = make_response(render_template(
'rq_dashboard/dashboard.html',
workers=Worker.all(),
queue=queue,
page=page,
queues=get_all_queues(),
rq_url_prefix=url_for('.overview')
))
r.headers.set('Cache-Control', 'no-store')
return r
def test_retry_worker_as_only_worker():
q = Queue()
rw = RetryWorker([q], retry_config=dict(
max_tries=4, maint_interval=0, delays=[]))
job = q.enqueue(error_fun)
for _ in range(1, 20):
rw.work(burst=True)
job.refresh()
assert q.count == 0
assert get_failed_queue().count == 0
assert Queue('dead_letter_queue').count == 1
assert job.meta['tries'] == 4
def test_no_rq_scheduler_falls_back_to_immediate_retry():
q = Queue()
q2 = Queue('not_used')
w = Worker([q])
rw = get_retry_worker()([q2], retry_config=dict(
max_tries=2, retry_delays=[1], maint_interval=0))
# run job that will fail
job = q.enqueue(error_fun)
w.work(burst=True)
assert q.count == 0
assert get_failed_queue().count == 1
# run retry worker
rw.work(burst=True)
# job should be requeued since rq_scheduler cannot be imported
assert q.count == 1
assert get_failed_queue().count == 0
assert Queue('dead_letter_queue').count == 0
job.refresh()
assert job.meta['tries'] == 2
def requeue():
"""Requeue all failed jobs in failed queue"""
failed_queue = get_failed_queue()
job_ids = failed_queue.job_ids
click.echo('Requeue failed jobs: {}'.format(len(job_ids)))
requeue_failed_num = 0
with click.progressbar(job_ids) as job_bar:
for job_id in job_bar:
try:
failed_queue.requeue(job_id)
except InvalidJobOperationError:
requeue_failed_num += 1
click.secho('Requeue failed: {}'.format(
requeue_failed_num), fg='red')
def get_context_data(self, **kwargs):
ctx = super(JobDetails, self).get_context_data(**kwargs)
try:
job = Job.fetch(self.kwargs['job'], connection=self.connection)
except NoSuchJobError:
raise Http404
if job.is_failed:
queue = get_failed_queue(connection=self.connection)
else:
queue = Queue(job.origin, connection=self.connection)
ctx.update({
'job': serialize_job(job),
'queue': queue,
'title': _('Job %s') % job.id,
})
return ctx
def test_requeue_job(self):
"""Requeueing existing jobs."""
job = Job.create(func=div_by_zero, args=(1, 2, 3))
job.origin = 'fake'
job.save()
get_failed_queue().quarantine(job, Exception('Some fake error')) # noqa
self.assertEqual(Queue.all(), [get_failed_queue()]) # noqa
self.assertEquals(get_failed_queue().count, 1)
get_failed_queue().requeue(job.id)
self.assertEquals(get_failed_queue().count, 0)
self.assertEquals(Queue('fake').count, 1)
def requeue_job_view(job_id):
# Just try both failed queues... don't care about efficiency for single job retries
timeout_queue = RqTimeoutQueue()
failed_queue = get_failed_queue()
try:
failed_queue.requeue(job_id)
except:
pass
try:
timeout_queue.requeue(job_id)
except:
pass
return dict(status='OK')
