def test_job_not_failed(self): # log admin user self._logSuperuserIn() # case 1: job didn't go through RQ-Scheduler, but directly to Queue job1 = self.queue.enqueue(dummy_job) rqjob1 = RQJob.objects.create(job_id=job1.id, trigger=self.trigger) Job.fetch(job1.id, connection=self.scheduler.connection) # no error with self.connection.pipeline() as pipe: pipe.watch(self.scheduler.scheduled_jobs_key) self.assertIsNone(pipe.zscore(self.scheduler.scheduled_jobs_key, job1.id)) url = reverse("admin:autoemails_rqjob_retry", args=[rqjob1.pk]) rv = self.client.post(url, follow=True) self.assertIn( "You cannot re-try a non-failed job.", rv.content.decode("utf-8"), ) # case 2: job is no longer in the RQ-Scheduler queue, but it was there! job2 = self.scheduler.enqueue_in( timedelta(minutes=5), dummy_job, ) rqjob2 = RQJob.objects.create(job_id=job2.id, trigger=self.trigger) # move job to the queue so it's executed self.scheduler.enqueue_job(job2) Job.fetch(job2.id, connection=self.scheduler.connection) # no error url = reverse("admin:autoemails_rqjob_retry", args=[rqjob2.pk]) rv = self.client.post(url, follow=True) self.assertIn( "You cannot re-try a non-failed job.", rv.content.decode("utf-8"), )
def test_job_rescheduled_correctly(self): # log admin user self._logSuperuserIn() job = self.scheduler.enqueue_in( timedelta(minutes=60), dummy_job, ) rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger) Job.fetch(job.id, connection=self.scheduler.connection) # no error url = reverse("admin:autoemails_rqjob_reschedule", args=[rqjob.pk]) scheduled_execution = datetime.now(tz=timezone.utc) + timedelta( days=1, minutes=15 ) scheduled_execution = scheduled_execution.replace(microsecond=0) payload = { "scheduled_execution": scheduled_execution, "scheduled_execution_0": f"{scheduled_execution:%Y-%m-%d}", "scheduled_execution_1": f"{scheduled_execution:%H:%M:%S}", } rv = self.client.post(url, payload, follow=True) self.assertIn( f"The job {job.id} was rescheduled to {scheduled_execution}", rv.content.decode("utf-8"), ) for _job, time in self.scheduler.get_jobs(with_times=True): if _job.id == job.id: scheduled = to_unix(datetime.utcnow() + timedelta(days=1, minutes=15)) epochtime = to_unix(time) self.assertAlmostEqual(epochtime, scheduled, delta=60) # +- 60s
def test_job_template_updated_correctly(self): # log admin user self._logSuperuserIn() action = NewInstructorAction( self.trigger, objects={ "event": self.event, "task": self.task }, ) job = self.scheduler.enqueue_in( timedelta(minutes=60), action, ) rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger) Job.fetch(job.id, connection=self.scheduler.connection) # no error url = reverse('admin:autoemails_rqjob_edit_template', args=[rqjob.pk]) payload = { 'template': self.new_template, } rv = self.client.post(url, payload, follow=True) self.assertIn( f'The job {job.id} template was updated', rv.content.decode('utf-8'), ) job.refresh() self.assertEqual(job.instance.template.body_template, "Welcome to AMY!")
def test_no_such_job(self): # log admin user self._logSuperuserIn() with self.assertRaises(NoSuchJobError): Job.fetch(self.rqjob.job_id, connection=self.scheduler.connection) url = reverse("admin:autoemails_rqjob_cancel", args=[self.rqjob.pk]) rv = self.client.post(url, follow=True) self.assertIn( "The corresponding job in Redis was probably already executed", rv.content.decode("utf-8"), )
def test_enqueued_job_cancelled(self): """Ensure enqueued job is successfully cancelled.""" # log admin user self._logSuperuserIn() # enqueue a job to run in future job = self.scheduler.enqueue_in( timedelta(minutes=5), dummy_job, ) rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger) # fetch job data job = Job.fetch(rqjob.job_id, connection=self.scheduler.connection) # `None` status is characteristic to scheduler-queued jobs. # Jobs added to the queue without scheduler will have different # status. self.assertEqual(job.get_status(), None) # the job is in scheduler's queue with self.connection.pipeline() as pipe: pipe.watch(self.scheduler.scheduled_jobs_key) # job in scheduler self.assertIsNotNone( pipe.zscore(self.scheduler.scheduled_jobs_key, job.id)) # cancel the job url = reverse('admin:autoemails_rqjob_cancel', args=[rqjob.pk]) rv = self.client.post(url, follow=True) self.assertIn( f'The job {rqjob.job_id} was cancelled.', rv.content.decode('utf-8'), ) # the job is no longer in scheduler's queue with self.connection.pipeline() as pipe: pipe.watch(self.scheduler.scheduled_jobs_key) # job in scheduler self.assertIsNone( pipe.zscore(self.scheduler.scheduled_jobs_key, job.id)) # job status updated rqjob.refresh_from_db() self.assertEqual(rqjob.status, "cancelled") # job data still available Job.fetch(rqjob.job_id, connection=self.scheduler.connection) # ...but nothing is scheduled self.assertEqual(self.scheduler.count(), 0)
def retry(self, request, object_id): """Fetch job and re-try to execute it.""" rqjob = get_object_or_404(RQJob, id=object_id) logger.debug(f"Re-trying job {rqjob.job_id}...") link = reverse("admin:autoemails_rqjob_preview", args=[object_id]) # fetch job try: job = Job.fetch(rqjob.job_id, connection=scheduler.connection) logger.debug(f"Job {rqjob.job_id} fetched") except NoSuchJobError: logger.debug(f"Job {rqjob.job_id} unavailable") messages.warning( request, "The corresponding job in Redis was probably already executed.", ) return redirect(link) if job.is_failed: job.requeue() logger.debug(f"Job {rqjob.job_id} retried. Will run shortly.") messages.info( request, f"The job {rqjob.job_id} was requeued. It will be run shortly.", ) else: logger.debug( f"Job {rqjob.job_id} can't be retried, because it was successful." ) messages.warning(request, "You cannot re-try a non-failed job.") return redirect(link)
def test_job_not_in_scheduled_jobs_queue(self): # log admin user self._logSuperuserIn() # case 1: job didn't go through RQ-Scheduler, but directly to Queue job1 = self.queue.enqueue(dummy_job) rqjob1 = RQJob.objects.create(job_id=job1.id, trigger=self.trigger) Job.fetch(job1.id, connection=self.scheduler.connection) # no error with self.connection.pipeline() as pipe: pipe.watch(self.scheduler.scheduled_jobs_key) self.assertIsNone(pipe.zscore(self.scheduler.scheduled_jobs_key, job1.id)) url = reverse("admin:autoemails_rqjob_reschedule", args=[rqjob1.pk]) scheduled_execution = datetime.now(tz=timezone.utc) + timedelta( days=1, minutes=15 ) scheduled_execution = scheduled_execution.replace(microsecond=0) payload = { "scheduled_execution": scheduled_execution, "scheduled_execution_0": f"{scheduled_execution:%Y-%m-%d}", "scheduled_execution_1": f"{scheduled_execution:%H:%M:%S}", } rv = self.client.post(url, payload, follow=True) self.assertIn( f"The job {job1.id} was not rescheduled. It is probably " "already executing or has recently executed", rv.content.decode("utf-8"), ) # case 2: job is no longer in the RQ-Scheduler queue, but it was there! job2 = self.scheduler.enqueue_in( timedelta(minutes=5), dummy_job, ) rqjob2 = RQJob.objects.create(job_id=job2.id, trigger=self.trigger) # move job to the queue so it's executed self.scheduler.enqueue_job(job2) Job.fetch(job2.id, connection=self.scheduler.connection) # no error url = reverse("admin:autoemails_rqjob_reschedule", args=[rqjob2.pk]) rv = self.client.post(url, payload, follow=True) self.assertIn( f"The job {job2.id} was not rescheduled. It is probably " "already executing or has recently executed", rv.content.decode("utf-8"), )
def test_job_executed(self): """Ensure executed job is discovered.""" # log admin user self._logSuperuserIn() # enqueue and then create an RQJob job = self.queue.enqueue(dummy_job) rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger) Job.fetch(job.id, connection=self.scheduler.connection) # no error with self.connection.pipeline() as pipe: pipe.watch(self.scheduler.scheduled_jobs_key) # no jobs in scheduler self.assertIsNone(pipe.zscore(self.scheduler.scheduled_jobs_key, job.id)) url = reverse("admin:autoemails_rqjob_cancel", args=[rqjob.pk]) rv = self.client.post(url, follow=True) self.assertIn( "Job has unknown status or was already executed.", rv.content.decode("utf-8"), )
def test_job_rescheduled_correctly(self): # log admin user self._logSuperuserIn() job = self.scheduler.enqueue_in( timedelta(minutes=60), dummy_job, ) rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger) Job.fetch(job.id, connection=self.scheduler.connection) # no error url = reverse('admin:autoemails_rqjob_sendnow', args=[rqjob.pk]) rv = self.client.post(url, follow=True) self.assertIn( f'The job {job.id} was rescheduled to now.', rv.content.decode('utf-8'), ) for _job, time in self.scheduler.get_jobs(with_times=True): if _job.id == job.id: now = to_unix(datetime.utcnow()) epochtime = to_unix(time) self.assertAlmostEqual(epochtime, now, delta=60) # +- 60s
def test_job_not_in_scheduled_jobs_queue(self): # log admin user self._logSuperuserIn() # case 1: job didn't go through RQ-Scheduler, but directly to Queue job1 = self.queue.enqueue(dummy_job) rqjob1 = RQJob.objects.create(job_id=job1.id, trigger=self.trigger) Job.fetch(job1.id, connection=self.scheduler.connection) # no error with self.connection.pipeline() as pipe: pipe.watch(self.scheduler.scheduled_jobs_key) self.assertIsNone( pipe.zscore(self.scheduler.scheduled_jobs_key, job1.id)) url = reverse('admin:autoemails_rqjob_edit_template', args=[rqjob1.pk]) payload = { 'template': self.new_template, } rv = self.client.post(url, payload, follow=True) self.assertIn( f"The job {job1.id} template cannot be updated.", rv.content.decode('utf-8'), ) # case 2: job is no longer in the RQ-Scheduler queue, but it was there! job2 = self.scheduler.enqueue_in( timedelta(minutes=5), dummy_job, ) rqjob2 = RQJob.objects.create(job_id=job2.id, trigger=self.trigger) # move job to the queue so it's executed self.scheduler.enqueue_job(job2) Job.fetch(job2.id, connection=self.scheduler.connection) # no error url = reverse('admin:autoemails_rqjob_edit_template', args=[rqjob2.pk]) rv = self.client.post(url, payload, follow=True) self.assertIn( f"The job {job2.id} template cannot be updated.", rv.content.decode('utf-8'), )
def edit_template(self, request, object_id): """Edit email template of a scheduled job.""" rqjob = get_object_or_404(RQJob, id=object_id) logger.debug(f"Editing job email template {rqjob.job_id}...") link = reverse("admin:autoemails_rqjob_preview", args=[object_id]) # fetch job try: job = Job.fetch(rqjob.job_id, connection=scheduler.connection) logger.debug(f"Job {rqjob.job_id} fetched") except NoSuchJobError: logger.debug(f"Job {rqjob.job_id} unavailable") messages.warning( request, "The corresponding job in Redis was probably already executed.", ) return redirect(link) if request.method == "POST": form = TemplateForm(request.POST) if form.is_valid(): new_tmpl = form.cleaned_data["template"] try: inst = job.instance inst.template.body_template = new_tmpl job.instance = inst job.save() logger.debug(f"Job {rqjob.job_id} template updated") messages.info( request, f"The job {rqjob.job_id} template was updated.", ) except AttributeError: logger.debug( f"Job {rqjob.job_id} template can't be updated.") messages.warning( request, f"The job {rqjob.job_id} template cannot be updated.", ) else: messages.warning(request, "Please fix errors below.") return redirect(link)
def cancel(self, request, object_id): """Fetch job and re-try to execute it.""" rqjob = get_object_or_404(RQJob, id=object_id) logger.debug(f"Cancelling job {rqjob.job_id}...") link = reverse("admin:autoemails_rqjob_preview", args=[object_id]) # fetch job try: job = Job.fetch(rqjob.job_id, connection=scheduler.connection) logger.debug(f"Job {rqjob.job_id} fetched") except NoSuchJobError: logger.debug(f"Job {rqjob.job_id} unavailable") messages.warning( request, "The corresponding job in Redis was probably already executed.", ) return redirect(link) if job.is_queued or not job.get_status(): job.cancel() # for "pure" jobs scheduler.cancel(job) # for scheduler-based jobs rqjob.status = "cancelled" rqjob.save() logger.debug(f"Job {rqjob.job_id} was cancelled.") messages.info(request, f"The job {rqjob.job_id} was cancelled.") elif job.is_started: # Right now we don't know how to test a started job, so we simply # don't allow such jobs to be cancelled. logger.debug( f"Job {rqjob.job_id} has started and cannot be cancelled.") messages.warning( request, f"Job {rqjob.job_id} has started and cannot be cancelled.") elif job.get_status() not in ("", None): logger.debug( f"Job {rqjob.job_id} has unknown status or was already executed." ) messages.warning( request, "Job has unknown status or was already executed.") return redirect(link)
def reschedule(self, request, object_id): """Change scheduled execution time to a different timestamp.""" rqjob = get_object_or_404(RQJob, id=object_id) logger.debug(f"Rescheduling job {rqjob.job_id}...") link = reverse("admin:autoemails_rqjob_preview", args=[object_id]) # fetch job try: job = Job.fetch(rqjob.job_id, connection=scheduler.connection) logger.debug(f"Job {rqjob.job_id} fetched") except NoSuchJobError: logger.debug(f"Job {rqjob.job_id} unavailable") messages.warning( request, "The corresponding job in Redis was probably already executed.", ) return redirect(link) if request.method == "POST": form = RescheduleForm(request.POST) if form.is_valid(): new_exec = form.cleaned_data["scheduled_execution"] try: scheduler.change_execution_time(job, new_exec) logger.debug(f"Job {rqjob.job_id} rescheduled") messages.info( request, f"The job {rqjob.job_id} was rescheduled to {new_exec}.", ) except ValueError: logger.debug( f"Job {rqjob.job_id} could not be rescheduled.") messages.warning( request, f"The job {rqjob.job_id} was not " "rescheduled. It is probably already " "executing or has recently executed.", ) else: messages.warning(request, "Please fix errors below.") return redirect(link)
def reschedule_now(self, request, object_id): """Reschedule an existing job so it executes now (+/- refresh time delta, about 1 minute in default settings).""" rqjob = get_object_or_404(RQJob, id=object_id) logger.debug( f"Executing job {rqjob.job_id} now (scheduling to +- 1min)...") link = reverse("admin:autoemails_rqjob_preview", args=[object_id]) # fetch job try: job = Job.fetch(rqjob.job_id, connection=scheduler.connection) logger.debug(f"Job {rqjob.job_id} fetched") except NoSuchJobError: logger.debug(f"Job {rqjob.job_id} unavailable") messages.warning( request, "The corresponding job in Redis was probably already executed.", ) return redirect(link) # new scheduled time: now (in UTC) now_utc = datetime.utcnow() try: scheduler.change_execution_time(job, now_utc) logger.debug(f"Job {rqjob.job_id} rescheduled to now") messages.info(request, f"The job {rqjob.job_id} was rescheduled to now.") except ValueError: logger.debug(f"Job {rqjob.job_id} could not be rescheduled.") messages.warning( request, f"The job {rqjob.job_id} was not " "rescheduled. It is probably already " "executing or has recently executed.", ) return redirect(link)
def testActionRemove(self): trigger = self.trigger task = self.task job_ids = MagicMock() # it will mock a QuerySet # Define a special class inheriting from the mixin we're about to test # so that we can (indirectly?) test the mixin itself. In some cases # the mock mechanism will have to be used, because - again - we can # only indirectly test the behavior of `action_remove()`. class MockView(ActionManageMixin): def __init__(self, connection, queue, scheduler, *args, **kwargs): super().__init__(*args, **kwargs) self.object = task self.logger = MagicMock() self.connection = connection self.queue = queue self.scheduler = scheduler def get_logger(self): return self.logger def get_scheduler(self): return self.scheduler def get_redis_connection(self): return self.connection def get_triggers(self): objs = [trigger] triggers = MagicMock() triggers.__iter__.return_value = iter(objs) triggers.count.return_value = len(objs) return triggers def objects(self): return dict(task=self.object, event=self.object.event) @property def request(self): # fake request created thanks to RequestFactory from Django # Test Client req = RequestFactory().post('/tasks/create') return req def get_jobs(self, as_id_list=True): if not as_id_list: raise NotImplementedError() return job_ids view = MockView(self.connection, self.queue, self.scheduler) # assertions before the view action is invoked self.assertEqual(self.scheduler.count(), 0) self.assertEqual(RQJob.objects.count(), 0) # view action invoke - it schedules a job view.action_add(NewInstructorAction) # additionally enqueue (as opposite to schedule) a blocking job enqueued_job = self.queue.enqueue(dummy_job) self.assertTrue(enqueued_job.is_finished) # ensure both a new Job and a corresponding RQJob were created self.assertEqual(self.scheduler.count(), 1) self.assertEqual(RQJob.objects.count(), 1) # ensure it's the same job job = next(self.scheduler.get_jobs()) rqjob = RQJob.objects.first() self.assertEqual(job.get_id(), rqjob.job_id) # mock a Query Set # previously enqueued job is added here so that the action_remove # interface is mocked into removing it from the enqueued jobs real_job_ids = [job.get_id(), enqueued_job.id] job_ids.__iter__.return_value = iter(real_job_ids) job_ids.count.return_value = len(real_job_ids) # invoke action_remove view.action_remove(NewInstructorAction) # ensure there are no scheduled jobs nor RQJob objects self.assertEqual(self.scheduler.count(), 0) self.assertEqual(RQJob.objects.count(), 0) # ensure the previously enqueued job is no longer available with self.assertRaises(NoSuchJobError): Job.fetch(enqueued_job.id, connection=self.connection) # logger.debug is called 6 times (for action_add) and 6 times # (for action_remove) self.assertEqual(view.get_logger().debug.call_count, 6 + 6)
def preview(self, request, object_id): """Show job + email preview (all details and statuses).""" rqjob = get_object_or_404(RQJob, id=object_id) logger.debug(f"Previewing job {rqjob.job_id}...") try: job = Job.fetch(rqjob.job_id, connection=scheduler.connection) job_scheduled = scheduled_execution_time(job.get_id(), scheduler) instance = job.instance status = check_status(job) logger.debug(f"Job {rqjob.job_id} fetched") # the job may not exist anymore, then we can't retrieve any data except NoSuchJobError: job = None job_scheduled = None instance = None status = None trigger = None template = None email = None adn_context = None logger.debug(f"Job {rqjob.job_id} unavailable") # we can try and read properties else: try: trigger = instance.trigger template = instance.template email = instance._email() adn_context = instance.context except AttributeError: trigger = None template = None email = None adn_context = None reschedule_form = None if job and not job.is_failed: now_utc = datetime.utcnow() + timedelta(minutes=10) reschedule_form = RescheduleForm( initial=dict(scheduled_execution=job_scheduled or now_utc) ) template_form = None if instance and job and not job.is_failed: template_form = TemplateForm( initial=dict(template=instance.template.body_template) ) # find prev / next RQJob in the list previous = RQJob.objects.filter(pk__lt=rqjob.pk).order_by("id").last() next_ = RQJob.objects.filter(pk__gt=rqjob.pk).order_by("id").first() context = dict( self.admin_site.each_context(request), cl=self.get_changelist_instance(request), title=f"Preview {rqjob}", rqjob=rqjob, job=job, job_scheduled=job_scheduled, instance=instance, status=status, trigger=trigger, template=template, email=email, adn_context=adn_context, reschedule_form=reschedule_form, template_form=template_form, previous=previous, next=next_, ) return TemplateResponse(request, "rqjob_preview.html", context)