def test_enqueue_dependents_on_multiple_queues(self):
"""Enqueueing dependent jobs on multiple queues pushes jobs in the queues
and removes them from DeferredJobRegistry for each different queue."""
q_1 = Queue("queue_1")
q_2 = Queue("queue_2")
parent_job = Job.create(func=say_hello)
parent_job.save()
job_1 = q_1.enqueue(say_hello, depends_on=parent_job)
job_2 = q_2.enqueue(say_hello, depends_on=parent_job)
# Each queue has its own DeferredJobRegistry
registry_1 = DeferredJobRegistry(q_1.name, connection=self.testconn)
self.assertEqual(
set(registry_1.get_job_ids()),
set([job_1.id])
)
registry_2 = DeferredJobRegistry(q_2.name, connection=self.testconn)
self.assertEqual(
set(registry_2.get_job_ids()),
set([job_2.id])
)
# After dependents is enqueued, job_1 on queue_1 and
# job_2 should be in queue_2
self.assertEqual(q_1.job_ids, [])
self.assertEqual(q_2.job_ids, [])
q_1.enqueue_dependents(parent_job)
q_2.enqueue_dependents(parent_job)
self.assertEqual(set(q_1.job_ids), set([job_1.id]))
self.assertEqual(set(q_2.job_ids), set([job_2.id]))
self.assertFalse(self.testconn.exists(parent_job.dependents_key))
# DeferredJobRegistry should also be empty
self.assertEqual(registry_1.get_job_ids(), [])
self.assertEqual(registry_2.get_job_ids(), [])
def test_enqueue_dependents_on_multiple_queues(self):
"""Enqueueing dependent jobs on multiple queues pushes jobs in the queues
and removes them from DeferredJobRegistry for each different queue."""
q_1 = Queue("queue_1")
q_2 = Queue("queue_2")
parent_job = Job.create(func=say_hello)
parent_job.save()
job_1 = q_1.enqueue(say_hello, depends_on=parent_job)
job_2 = q_2.enqueue(say_hello, depends_on=parent_job)
# Each queue has its own DeferredJobRegistry
registry_1 = DeferredJobRegistry(q_1.name, connection=self.testconn)
self.assertEqual(set(registry_1.get_job_ids()), set([job_1.id]))
registry_2 = DeferredJobRegistry(q_2.name, connection=self.testconn)
self.assertEqual(set(registry_2.get_job_ids()), set([job_2.id]))
# After dependents is enqueued, job_1 on queue_1 and
# job_2 should be in queue_2
self.assertEqual(q_1.job_ids, [])
self.assertEqual(q_2.job_ids, [])
q_1.enqueue_dependents(parent_job)
q_2.enqueue_dependents(parent_job)
self.assertEqual(set(q_1.job_ids), set([job_1.id]))
self.assertEqual(set(q_2.job_ids), set([job_2.id]))
self.assertFalse(self.testconn.exists(parent_job.dependents_key))
# DeferredJobRegistry should also be empty
self.assertEqual(registry_1.get_job_ids(), [])
self.assertEqual(registry_2.get_job_ids(), [])
def test_enqueue_dependents(self):
"""Enqueueing dependent jobs pushes all jobs in the depends set to the queue
and removes them from DeferredJobQueue."""
q = Queue()
parent_job = Job.create(func=say_hello)
parent_job.save()
job_1 = q.enqueue(say_hello, depends_on=parent_job)
job_2 = q.enqueue(say_hello, depends_on=parent_job)
registry = DeferredJobRegistry(q.name, connection=self.testconn)
parent_job.set_status(JobStatus.FINISHED)
self.assertEqual(
set(registry.get_job_ids()),
set([job_1.id, job_2.id])
)
# After dependents is enqueued, job_1 and job_2 should be in queue
self.assertEqual(q.job_ids, [])
q.enqueue_dependents(parent_job)
self.assertEqual(set(q.job_ids), set([job_2.id, job_1.id]))
self.assertFalse(self.testconn.exists(parent_job.dependents_key))
# DeferredJobRegistry should also be empty
self.assertEqual(registry.get_job_ids(), [])
def get(self):
"""获取任务列表"""
args = request.args
job_status = args.get('job_status')
queue_name = args.get('queue_name')
if job_status not in Config.RQ_JOB_STATUS:
return {
'code': StatesCode.JOB_STATUS_NO_EXIST,
'message': '任务状态不存在！'
}
if queue_name not in Config.RQ_QUEUES_ALL:
return {'code': StatesCode.QUEUE_NOT_EXIST, 'message': '任务队列不存在！'}
job_list = []
if job_status == 'queued':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
job_list += queue_dict[queue_name].get_job_ids()
else:
job_list = queue_dict[queue_name].get_job_ids()
elif job_status == 'started':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
started_job_registry = StartedJobRegistry(
queue=queue_dict[queue_name])
job_list += started_job_registry.get_job_ids()
else:
started_job_registry = StartedJobRegistry(
queue=queue_dict[queue_name])
job_list = started_job_registry.get_job_ids()
elif job_status == 'finished':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
finished_job_registry = FinishedJobRegistry(
queue=queue_dict[queue_name])
job_list += finished_job_registry.get_job_ids()
else:
finished_job_registry = FinishedJobRegistry(
queue=queue_dict[queue_name])
job_list = finished_job_registry.get_job_ids()
elif job_status == 'failed':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
failed_job_registry = FailedJobRegistry(
queue=queue_dict[queue_name])
job_list += failed_job_registry.get_job_ids()
else:
failed_job_registry = FailedJobRegistry(
queue=queue_dict[queue_name])
job_list = failed_job_registry.get_job_ids()
elif job_status == 'deferred':
if queue_name == 'all':
for queue_name in Config.RQ_QUEUES:
deferred_job_registry = DeferredJobRegistry(
queue=queue_dict[queue_name])
job_list += deferred_job_registry.get_job_ids()
else:
deferred_job_registry = DeferredJobRegistry(
queue=queue_dict[queue_name])
job_list = deferred_job_registry.get_job_ids()
return {'code': StatesCode.SUCCESS, 'data': job_list}
def test_register_dependency(self):
"""Ensure job creation and deletion works properly with DeferredJobRegistry."""
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
job2 = queue.enqueue(say_hello, depends_on=job)
registry = DeferredJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_job_ids(), [job2.id])
# When deleted, job removes itself from DeferredJobRegistry
job2.delete()
self.assertEqual(registry.get_job_ids(), [])
def test_register_dependency(self):
"""Ensure job creation and deletion works with DeferredJobRegistry."""
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
job2 = queue.enqueue(say_hello, depends_on=job)
registry = DeferredJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_job_ids(), [job2.id])
# When deleted, job removes itself from DeferredJobRegistry
job2.delete()
self.assertEqual(registry.get_job_ids(), [])
def test_register_dependency(self):
"""Ensure dependency registration works properly."""
origin = 'some_queue'
registry = DeferredJobRegistry(origin, self.testconn)
job = Job.create(func=fixtures.say_hello, origin=origin)
job._dependency_id = 'id'
job.save()
self.assertEqual(registry.get_job_ids(), [])
job.register_dependency()
self.assertEqual(as_text(self.testconn.spop('rq:job:id:dependents')), job.id)
self.assertEqual(registry.get_job_ids(), [job.id])
def test_register_dependency(self):
"""Ensure dependency registration works properly."""
origin = 'some_queue'
registry = DeferredJobRegistry(origin, self.testconn)
job = Job.create(func=fixtures.say_hello, origin=origin)
job._dependency_id = 'id'
job.save()
self.assertEqual(registry.get_job_ids(), [])
job.register_dependency()
self.assertEqual(as_text(self.testconn.spop('rq:job:id:dependents')), job.id)
self.assertEqual(registry.get_job_ids(), [job.id])
def getDeferredExperiments(cls):
with Connection(redis.from_url(
current_app.config['REDIS_URL'])) as conn:
registry = DeferredJobRegistry('default', connection=conn)
return [
Job.fetch(id, connection=conn)
for id in registry.get_job_ids()
]
def test_enqueue_dependents(self):
"""Enqueueing dependent jobs pushes all jobs in the depends set to the queue
and removes them from DeferredJobQueue."""
q = Queue()
parent_job = Job.create(func=say_hello)
parent_job.save()
job_1 = q.enqueue(say_hello, depends_on=parent_job)
job_2 = q.enqueue(say_hello, depends_on=parent_job)
registry = DeferredJobRegistry(q.name, connection=self.testconn)
self.assertEqual(
set(registry.get_job_ids()),
set([job_1.id, job_2.id])
)
# After dependents is enqueued, job_1 and job_2 should be in queue
self.assertEqual(q.job_ids, [])
q.enqueue_dependents(parent_job)
self.assertEqual(set(q.job_ids), set([job_2.id, job_1.id]))
self.assertFalse(self.testconn.exists(parent_job.dependents_key))
# DeferredJobRegistry should also be empty
self.assertEqual(registry.get_job_ids(), [])
def test_enqueue_deferred(self):
"""Enqueuing a deferred job"""
q = Queue('example', async=False)
j = q.enqueue(say_hello, args=('John',), deferred=True)
# Job is deferred
self.assertEqual(j.get_status(), JobStatus.DEFERRED)
# Job is registered in appropriate registry
registry = DeferredJobRegistry(q.name, connection=self.testconn)
self.assertEqual(registry.get_job_ids(), [j.id])
# Job result isn't available though we're using a sync queue
self.assertIsNone(j.result)
# Freeing the job
j.perform()
def deferred_jobs(request, queue_index):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
registry = DeferredJobRegistry(queue.name, queue.connection)
items_per_page = 100
num_jobs = len(registry)
page = int(request.GET.get('page', 1))
jobs = []
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
job_ids = registry.get_job_ids(offset, items_per_page)
for job_id in job_ids:
try:
jobs.append(Job.fetch(job_id, connection=queue.connection))
except NoSuchJobError:
pass
else:
page_range = []
###
# Custom logic here
for job in jobs:
use_actual_name(job)
##
context_data = {
'queue': queue,
'queue_index': queue_index,
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
'job_status': 'Deferred',
}
return render(request, 'django_rq/jobs.html', context_data)
def deferred_jobs(request, queue_index):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
registry = DeferredJobRegistry(queue.name, queue.connection)
items_per_page = 100
num_jobs = len(registry)
page = int(request.GET.get('page', 1))
jobs = []
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
job_ids = registry.get_job_ids(offset, items_per_page)
for job_id in job_ids:
try:
jobs.append(Job.fetch(job_id, connection=queue.connection))
except NoSuchJobError:
pass
else:
page_range = []
context_data = admin.site.each_context(request)
context_data.update({
'title': _("Deferred Jobs"),
'queue': queue,
'queue_index': queue_index,
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
'job_status': _('Deferred'),
})
return render(request, 'django_rq/jobs.html', context_data)
def deferred_jobs(request, queue_index):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
registry = DeferredJobRegistry(queue.name, queue.connection)
items_per_page = 100
num_jobs = len(registry)
page = int(request.GET.get('page', 1))
jobs = []
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
job_ids = registry.get_job_ids(offset, offset + items_per_page - 1)
for job_id in job_ids:
try:
jobs.append(Job.fetch(job_id, connection=queue.connection))
except NoSuchJobError:
pass
else:
page_range = []
context_data = {
'queue': queue,
'queue_index': queue_index,
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
'job_status': 'Deferred',
}
return render(request, 'django_rq/jobs.html', context_data)
