def test_enqueue_dependents_on_multiple_queues(self): """Enqueueing dependent jobs on multiple queues pushes jobs in the queues and removes them from DeferredJobRegistry for each different queue.""" q_1 = Queue("queue_1") q_2 = Queue("queue_2") parent_job = Job.create(func=say_hello) parent_job.save() job_1 = q_1.enqueue(say_hello, depends_on=parent_job) job_2 = q_2.enqueue(say_hello, depends_on=parent_job) # Each queue has its own DeferredJobRegistry registry_1 = DeferredJobRegistry(q_1.name, connection=self.testconn) self.assertEqual( set(registry_1.get_job_ids()), set([job_1.id]) ) registry_2 = DeferredJobRegistry(q_2.name, connection=self.testconn) self.assertEqual( set(registry_2.get_job_ids()), set([job_2.id]) ) # After dependents is enqueued, job_1 on queue_1 and # job_2 should be in queue_2 self.assertEqual(q_1.job_ids, []) self.assertEqual(q_2.job_ids, []) q_1.enqueue_dependents(parent_job) q_2.enqueue_dependents(parent_job) self.assertEqual(set(q_1.job_ids), set([job_1.id])) self.assertEqual(set(q_2.job_ids), set([job_2.id])) self.assertFalse(self.testconn.exists(parent_job.dependents_key)) # DeferredJobRegistry should also be empty self.assertEqual(registry_1.get_job_ids(), []) self.assertEqual(registry_2.get_job_ids(), [])
def test_enqueue_dependents_on_multiple_queues(self): """Enqueueing dependent jobs on multiple queues pushes jobs in the queues and removes them from DeferredJobRegistry for each different queue.""" q_1 = Queue("queue_1") q_2 = Queue("queue_2") parent_job = Job.create(func=say_hello) parent_job.save() job_1 = q_1.enqueue(say_hello, depends_on=parent_job) job_2 = q_2.enqueue(say_hello, depends_on=parent_job) # Each queue has its own DeferredJobRegistry registry_1 = DeferredJobRegistry(q_1.name, connection=self.testconn) self.assertEqual(set(registry_1.get_job_ids()), set([job_1.id])) registry_2 = DeferredJobRegistry(q_2.name, connection=self.testconn) self.assertEqual(set(registry_2.get_job_ids()), set([job_2.id])) # After dependents is enqueued, job_1 on queue_1 and # job_2 should be in queue_2 self.assertEqual(q_1.job_ids, []) self.assertEqual(q_2.job_ids, []) q_1.enqueue_dependents(parent_job) q_2.enqueue_dependents(parent_job) self.assertEqual(set(q_1.job_ids), set([job_1.id])) self.assertEqual(set(q_2.job_ids), set([job_2.id])) self.assertFalse(self.testconn.exists(parent_job.dependents_key)) # DeferredJobRegistry should also be empty self.assertEqual(registry_1.get_job_ids(), []) self.assertEqual(registry_2.get_job_ids(), [])
def test_enqueue_dependents(self): """Enqueueing dependent jobs pushes all jobs in the depends set to the queue and removes them from DeferredJobQueue.""" q = Queue() parent_job = Job.create(func=say_hello) parent_job.save() job_1 = q.enqueue(say_hello, depends_on=parent_job) job_2 = q.enqueue(say_hello, depends_on=parent_job) registry = DeferredJobRegistry(q.name, connection=self.testconn) parent_job.set_status(JobStatus.FINISHED) self.assertEqual( set(registry.get_job_ids()), set([job_1.id, job_2.id]) ) # After dependents is enqueued, job_1 and job_2 should be in queue self.assertEqual(q.job_ids, []) q.enqueue_dependents(parent_job) self.assertEqual(set(q.job_ids), set([job_2.id, job_1.id])) self.assertFalse(self.testconn.exists(parent_job.dependents_key)) # DeferredJobRegistry should also be empty self.assertEqual(registry.get_job_ids(), [])
def get(self): """获取任务列表""" args = request.args job_status = args.get('job_status') queue_name = args.get('queue_name') if job_status not in Config.RQ_JOB_STATUS: return { 'code': StatesCode.JOB_STATUS_NO_EXIST, 'message': '任务状态不存在!' } if queue_name not in Config.RQ_QUEUES_ALL: return {'code': StatesCode.QUEUE_NOT_EXIST, 'message': '任务队列不存在!'} job_list = [] if job_status == 'queued': if queue_name == 'all': for queue_name in Config.RQ_QUEUES: job_list += queue_dict[queue_name].get_job_ids() else: job_list = queue_dict[queue_name].get_job_ids() elif job_status == 'started': if queue_name == 'all': for queue_name in Config.RQ_QUEUES: started_job_registry = StartedJobRegistry( queue=queue_dict[queue_name]) job_list += started_job_registry.get_job_ids() else: started_job_registry = StartedJobRegistry( queue=queue_dict[queue_name]) job_list = started_job_registry.get_job_ids() elif job_status == 'finished': if queue_name == 'all': for queue_name in Config.RQ_QUEUES: finished_job_registry = FinishedJobRegistry( queue=queue_dict[queue_name]) job_list += finished_job_registry.get_job_ids() else: finished_job_registry = FinishedJobRegistry( queue=queue_dict[queue_name]) job_list = finished_job_registry.get_job_ids() elif job_status == 'failed': if queue_name == 'all': for queue_name in Config.RQ_QUEUES: failed_job_registry = FailedJobRegistry( queue=queue_dict[queue_name]) job_list += failed_job_registry.get_job_ids() else: failed_job_registry = FailedJobRegistry( queue=queue_dict[queue_name]) job_list = failed_job_registry.get_job_ids() elif job_status == 'deferred': if queue_name == 'all': for queue_name in Config.RQ_QUEUES: deferred_job_registry = DeferredJobRegistry( queue=queue_dict[queue_name]) job_list += deferred_job_registry.get_job_ids() else: deferred_job_registry = DeferredJobRegistry( queue=queue_dict[queue_name]) job_list = deferred_job_registry.get_job_ids() return {'code': StatesCode.SUCCESS, 'data': job_list}
def test_register_dependency(self): """Ensure job creation and deletion works properly with DeferredJobRegistry.""" queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) job2 = queue.enqueue(say_hello, depends_on=job) registry = DeferredJobRegistry(connection=self.testconn) self.assertEqual(registry.get_job_ids(), [job2.id]) # When deleted, job removes itself from DeferredJobRegistry job2.delete() self.assertEqual(registry.get_job_ids(), [])
def test_register_dependency(self): """Ensure job creation and deletion works with DeferredJobRegistry.""" queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) job2 = queue.enqueue(say_hello, depends_on=job) registry = DeferredJobRegistry(connection=self.testconn) self.assertEqual(registry.get_job_ids(), [job2.id]) # When deleted, job removes itself from DeferredJobRegistry job2.delete() self.assertEqual(registry.get_job_ids(), [])
def test_register_dependency(self): """Ensure dependency registration works properly.""" origin = 'some_queue' registry = DeferredJobRegistry(origin, self.testconn) job = Job.create(func=fixtures.say_hello, origin=origin) job._dependency_id = 'id' job.save() self.assertEqual(registry.get_job_ids(), []) job.register_dependency() self.assertEqual(as_text(self.testconn.spop('rq:job:id:dependents')), job.id) self.assertEqual(registry.get_job_ids(), [job.id])
def test_register_dependency(self): """Ensure dependency registration works properly.""" origin = 'some_queue' registry = DeferredJobRegistry(origin, self.testconn) job = Job.create(func=fixtures.say_hello, origin=origin) job._dependency_id = 'id' job.save() self.assertEqual(registry.get_job_ids(), []) job.register_dependency() self.assertEqual(as_text(self.testconn.spop('rq:job:id:dependents')), job.id) self.assertEqual(registry.get_job_ids(), [job.id])
def getDeferredExperiments(cls): with Connection(redis.from_url( current_app.config['REDIS_URL'])) as conn: registry = DeferredJobRegistry('default', connection=conn) return [ Job.fetch(id, connection=conn) for id in registry.get_job_ids() ]
def test_enqueue_dependents(self): """Enqueueing dependent jobs pushes all jobs in the depends set to the queue and removes them from DeferredJobQueue.""" q = Queue() parent_job = Job.create(func=say_hello) parent_job.save() job_1 = q.enqueue(say_hello, depends_on=parent_job) job_2 = q.enqueue(say_hello, depends_on=parent_job) registry = DeferredJobRegistry(q.name, connection=self.testconn) self.assertEqual( set(registry.get_job_ids()), set([job_1.id, job_2.id]) ) # After dependents is enqueued, job_1 and job_2 should be in queue self.assertEqual(q.job_ids, []) q.enqueue_dependents(parent_job) self.assertEqual(set(q.job_ids), set([job_2.id, job_1.id])) self.assertFalse(self.testconn.exists(parent_job.dependents_key)) # DeferredJobRegistry should also be empty self.assertEqual(registry.get_job_ids(), [])
def test_enqueue_deferred(self): """Enqueuing a deferred job""" q = Queue('example', async=False) j = q.enqueue(say_hello, args=('John',), deferred=True) # Job is deferred self.assertEqual(j.get_status(), JobStatus.DEFERRED) # Job is registered in appropriate registry registry = DeferredJobRegistry(q.name, connection=self.testconn) self.assertEqual(registry.get_job_ids(), [j.id]) # Job result isn't available though we're using a sync queue self.assertIsNone(j.result) # Freeing the job j.perform()
def deferred_jobs(request, queue_index): queue_index = int(queue_index) queue = get_queue_by_index(queue_index) registry = DeferredJobRegistry(queue.name, queue.connection) items_per_page = 100 num_jobs = len(registry) page = int(request.GET.get('page', 1)) jobs = [] if num_jobs > 0: last_page = int(ceil(num_jobs / items_per_page)) page_range = range(1, last_page + 1) offset = items_per_page * (page - 1) job_ids = registry.get_job_ids(offset, items_per_page) for job_id in job_ids: try: jobs.append(Job.fetch(job_id, connection=queue.connection)) except NoSuchJobError: pass else: page_range = [] ### # Custom logic here for job in jobs: use_actual_name(job) ## context_data = { 'queue': queue, 'queue_index': queue_index, 'jobs': jobs, 'num_jobs': num_jobs, 'page': page, 'page_range': page_range, 'job_status': 'Deferred', } return render(request, 'django_rq/jobs.html', context_data)
def deferred_jobs(request, queue_index): queue_index = int(queue_index) queue = get_queue_by_index(queue_index) registry = DeferredJobRegistry(queue.name, queue.connection) items_per_page = 100 num_jobs = len(registry) page = int(request.GET.get('page', 1)) jobs = [] if num_jobs > 0: last_page = int(ceil(num_jobs / items_per_page)) page_range = range(1, last_page + 1) offset = items_per_page * (page - 1) job_ids = registry.get_job_ids(offset, items_per_page) for job_id in job_ids: try: jobs.append(Job.fetch(job_id, connection=queue.connection)) except NoSuchJobError: pass else: page_range = [] context_data = admin.site.each_context(request) context_data.update({ 'title': _("Deferred Jobs"), 'queue': queue, 'queue_index': queue_index, 'jobs': jobs, 'num_jobs': num_jobs, 'page': page, 'page_range': page_range, 'job_status': _('Deferred'), }) return render(request, 'django_rq/jobs.html', context_data)
def deferred_jobs(request, queue_index): queue_index = int(queue_index) queue = get_queue_by_index(queue_index) registry = DeferredJobRegistry(queue.name, queue.connection) items_per_page = 100 num_jobs = len(registry) page = int(request.GET.get('page', 1)) jobs = [] if num_jobs > 0: last_page = int(ceil(num_jobs / items_per_page)) page_range = range(1, last_page + 1) offset = items_per_page * (page - 1) job_ids = registry.get_job_ids(offset, offset + items_per_page - 1) for job_id in job_ids: try: jobs.append(Job.fetch(job_id, connection=queue.connection)) except NoSuchJobError: pass else: page_range = [] context_data = { 'queue': queue, 'queue_index': queue_index, 'jobs': jobs, 'num_jobs': num_jobs, 'page': page, 'page_range': page_range, 'job_status': 'Deferred', } return render(request, 'django_rq/jobs.html', context_data)