def test_job_not_failed(self):
# log admin user
self._logSuperuserIn()
# case 1: job didn't go through RQ-Scheduler, but directly to Queue
job1 = self.queue.enqueue(dummy_job)
rqjob1 = RQJob.objects.create(job_id=job1.id, trigger=self.trigger)
Job.fetch(job1.id, connection=self.scheduler.connection) # no error
with self.connection.pipeline() as pipe:
pipe.watch(self.scheduler.scheduled_jobs_key)
self.assertIsNone(pipe.zscore(self.scheduler.scheduled_jobs_key, job1.id))
url = reverse("admin:autoemails_rqjob_retry", args=[rqjob1.pk])
rv = self.client.post(url, follow=True)
self.assertIn(
"You cannot re-try a non-failed job.",
rv.content.decode("utf-8"),
)
# case 2: job is no longer in the RQ-Scheduler queue, but it was there!
job2 = self.scheduler.enqueue_in(
timedelta(minutes=5),
dummy_job,
)
rqjob2 = RQJob.objects.create(job_id=job2.id, trigger=self.trigger)
# move job to the queue so it's executed
self.scheduler.enqueue_job(job2)
Job.fetch(job2.id, connection=self.scheduler.connection) # no error
url = reverse("admin:autoemails_rqjob_retry", args=[rqjob2.pk])
rv = self.client.post(url, follow=True)
self.assertIn(
"You cannot re-try a non-failed job.",
rv.content.decode("utf-8"),
)
def test_job_rescheduled_correctly(self):
# log admin user
self._logSuperuserIn()
job = self.scheduler.enqueue_in(
timedelta(minutes=60),
dummy_job,
)
rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger)
Job.fetch(job.id, connection=self.scheduler.connection) # no error
url = reverse("admin:autoemails_rqjob_reschedule", args=[rqjob.pk])
scheduled_execution = datetime.now(tz=timezone.utc) + timedelta(
days=1, minutes=15
)
scheduled_execution = scheduled_execution.replace(microsecond=0)
payload = {
"scheduled_execution": scheduled_execution,
"scheduled_execution_0": f"{scheduled_execution:%Y-%m-%d}",
"scheduled_execution_1": f"{scheduled_execution:%H:%M:%S}",
}
rv = self.client.post(url, payload, follow=True)
self.assertIn(
f"The job {job.id} was rescheduled to {scheduled_execution}",
rv.content.decode("utf-8"),
)
for _job, time in self.scheduler.get_jobs(with_times=True):
if _job.id == job.id:
scheduled = to_unix(datetime.utcnow() + timedelta(days=1, minutes=15))
epochtime = to_unix(time)
self.assertAlmostEqual(epochtime, scheduled, delta=60) # +- 60s
def test_job_template_updated_correctly(self):
# log admin user
self._logSuperuserIn()
action = NewInstructorAction(
self.trigger,
objects={
"event": self.event,
"task": self.task
},
)
job = self.scheduler.enqueue_in(
timedelta(minutes=60),
action,
)
rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger)
Job.fetch(job.id, connection=self.scheduler.connection) # no error
url = reverse('admin:autoemails_rqjob_edit_template', args=[rqjob.pk])
payload = {
'template': self.new_template,
}
rv = self.client.post(url, payload, follow=True)
self.assertIn(
f'The job {job.id} template was updated',
rv.content.decode('utf-8'),
)
job.refresh()
self.assertEqual(job.instance.template.body_template,
"Welcome to AMY!")
def test_no_such_job(self):
# log admin user
self._logSuperuserIn()
with self.assertRaises(NoSuchJobError):
Job.fetch(self.rqjob.job_id, connection=self.scheduler.connection)
url = reverse("admin:autoemails_rqjob_cancel", args=[self.rqjob.pk])
rv = self.client.post(url, follow=True)
self.assertIn(
"The corresponding job in Redis was probably already executed",
rv.content.decode("utf-8"),
)
def test_enqueued_job_cancelled(self):
"""Ensure enqueued job is successfully cancelled."""
# log admin user
self._logSuperuserIn()
# enqueue a job to run in future
job = self.scheduler.enqueue_in(
timedelta(minutes=5),
dummy_job,
)
rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger)
# fetch job data
job = Job.fetch(rqjob.job_id, connection=self.scheduler.connection)
# `None` status is characteristic to scheduler-queued jobs.
# Jobs added to the queue without scheduler will have different
# status.
self.assertEqual(job.get_status(), None)
# the job is in scheduler's queue
with self.connection.pipeline() as pipe:
pipe.watch(self.scheduler.scheduled_jobs_key)
# job in scheduler
self.assertIsNotNone(
pipe.zscore(self.scheduler.scheduled_jobs_key, job.id))
# cancel the job
url = reverse('admin:autoemails_rqjob_cancel', args=[rqjob.pk])
rv = self.client.post(url, follow=True)
self.assertIn(
f'The job {rqjob.job_id} was cancelled.',
rv.content.decode('utf-8'),
)
# the job is no longer in scheduler's queue
with self.connection.pipeline() as pipe:
pipe.watch(self.scheduler.scheduled_jobs_key)
# job in scheduler
self.assertIsNone(
pipe.zscore(self.scheduler.scheduled_jobs_key, job.id))
# job status updated
rqjob.refresh_from_db()
self.assertEqual(rqjob.status, "cancelled")
# job data still available
Job.fetch(rqjob.job_id, connection=self.scheduler.connection)
# ...but nothing is scheduled
self.assertEqual(self.scheduler.count(), 0)
def retry(self, request, object_id):
"""Fetch job and re-try to execute it."""
rqjob = get_object_or_404(RQJob, id=object_id)
logger.debug(f"Re-trying job {rqjob.job_id}...")
link = reverse("admin:autoemails_rqjob_preview", args=[object_id])
# fetch job
try:
job = Job.fetch(rqjob.job_id, connection=scheduler.connection)
logger.debug(f"Job {rqjob.job_id} fetched")
except NoSuchJobError:
logger.debug(f"Job {rqjob.job_id} unavailable")
messages.warning(
request,
"The corresponding job in Redis was probably already executed.",
)
return redirect(link)
if job.is_failed:
job.requeue()
logger.debug(f"Job {rqjob.job_id} retried. Will run shortly.")
messages.info(
request,
f"The job {rqjob.job_id} was requeued. It will be run shortly.",
)
else:
logger.debug(
f"Job {rqjob.job_id} can't be retried, because it was successful."
)
messages.warning(request, "You cannot re-try a non-failed job.")
return redirect(link)
def test_job_not_in_scheduled_jobs_queue(self):
# log admin user
self._logSuperuserIn()
# case 1: job didn't go through RQ-Scheduler, but directly to Queue
job1 = self.queue.enqueue(dummy_job)
rqjob1 = RQJob.objects.create(job_id=job1.id, trigger=self.trigger)
Job.fetch(job1.id, connection=self.scheduler.connection) # no error
with self.connection.pipeline() as pipe:
pipe.watch(self.scheduler.scheduled_jobs_key)
self.assertIsNone(pipe.zscore(self.scheduler.scheduled_jobs_key, job1.id))
url = reverse("admin:autoemails_rqjob_reschedule", args=[rqjob1.pk])
scheduled_execution = datetime.now(tz=timezone.utc) + timedelta(
days=1, minutes=15
)
scheduled_execution = scheduled_execution.replace(microsecond=0)
payload = {
"scheduled_execution": scheduled_execution,
"scheduled_execution_0": f"{scheduled_execution:%Y-%m-%d}",
"scheduled_execution_1": f"{scheduled_execution:%H:%M:%S}",
}
rv = self.client.post(url, payload, follow=True)
self.assertIn(
f"The job {job1.id} was not rescheduled. It is probably "
"already executing or has recently executed",
rv.content.decode("utf-8"),
)
# case 2: job is no longer in the RQ-Scheduler queue, but it was there!
job2 = self.scheduler.enqueue_in(
timedelta(minutes=5),
dummy_job,
)
rqjob2 = RQJob.objects.create(job_id=job2.id, trigger=self.trigger)
# move job to the queue so it's executed
self.scheduler.enqueue_job(job2)
Job.fetch(job2.id, connection=self.scheduler.connection) # no error
url = reverse("admin:autoemails_rqjob_reschedule", args=[rqjob2.pk])
rv = self.client.post(url, payload, follow=True)
self.assertIn(
f"The job {job2.id} was not rescheduled. It is probably "
"already executing or has recently executed",
rv.content.decode("utf-8"),
)
def test_job_executed(self):
"""Ensure executed job is discovered."""
# log admin user
self._logSuperuserIn()
# enqueue and then create an RQJob
job = self.queue.enqueue(dummy_job)
rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger)
Job.fetch(job.id, connection=self.scheduler.connection) # no error
with self.connection.pipeline() as pipe:
pipe.watch(self.scheduler.scheduled_jobs_key)
# no jobs in scheduler
self.assertIsNone(pipe.zscore(self.scheduler.scheduled_jobs_key, job.id))
url = reverse("admin:autoemails_rqjob_cancel", args=[rqjob.pk])
rv = self.client.post(url, follow=True)
self.assertIn(
"Job has unknown status or was already executed.",
rv.content.decode("utf-8"),
)
def test_job_rescheduled_correctly(self):
# log admin user
self._logSuperuserIn()
job = self.scheduler.enqueue_in(
timedelta(minutes=60),
dummy_job,
)
rqjob = RQJob.objects.create(job_id=job.id, trigger=self.trigger)
Job.fetch(job.id, connection=self.scheduler.connection) # no error
url = reverse('admin:autoemails_rqjob_sendnow', args=[rqjob.pk])
rv = self.client.post(url, follow=True)
self.assertIn(
f'The job {job.id} was rescheduled to now.',
rv.content.decode('utf-8'),
)
for _job, time in self.scheduler.get_jobs(with_times=True):
if _job.id == job.id:
now = to_unix(datetime.utcnow())
epochtime = to_unix(time)
self.assertAlmostEqual(epochtime, now, delta=60) # +- 60s
def test_job_not_in_scheduled_jobs_queue(self):
# log admin user
self._logSuperuserIn()
# case 1: job didn't go through RQ-Scheduler, but directly to Queue
job1 = self.queue.enqueue(dummy_job)
rqjob1 = RQJob.objects.create(job_id=job1.id, trigger=self.trigger)
Job.fetch(job1.id, connection=self.scheduler.connection) # no error
with self.connection.pipeline() as pipe:
pipe.watch(self.scheduler.scheduled_jobs_key)
self.assertIsNone(
pipe.zscore(self.scheduler.scheduled_jobs_key, job1.id))
url = reverse('admin:autoemails_rqjob_edit_template', args=[rqjob1.pk])
payload = {
'template': self.new_template,
}
rv = self.client.post(url, payload, follow=True)
self.assertIn(
f"The job {job1.id} template cannot be updated.",
rv.content.decode('utf-8'),
)
# case 2: job is no longer in the RQ-Scheduler queue, but it was there!
job2 = self.scheduler.enqueue_in(
timedelta(minutes=5),
dummy_job,
)
rqjob2 = RQJob.objects.create(job_id=job2.id, trigger=self.trigger)
# move job to the queue so it's executed
self.scheduler.enqueue_job(job2)
Job.fetch(job2.id, connection=self.scheduler.connection) # no error
url = reverse('admin:autoemails_rqjob_edit_template', args=[rqjob2.pk])
rv = self.client.post(url, payload, follow=True)
self.assertIn(
f"The job {job2.id} template cannot be updated.",
rv.content.decode('utf-8'),
)
def edit_template(self, request, object_id):
"""Edit email template of a scheduled job."""
rqjob = get_object_or_404(RQJob, id=object_id)
logger.debug(f"Editing job email template {rqjob.job_id}...")
link = reverse("admin:autoemails_rqjob_preview", args=[object_id])
# fetch job
try:
job = Job.fetch(rqjob.job_id, connection=scheduler.connection)
logger.debug(f"Job {rqjob.job_id} fetched")
except NoSuchJobError:
logger.debug(f"Job {rqjob.job_id} unavailable")
messages.warning(
request,
"The corresponding job in Redis was probably already executed.",
)
return redirect(link)
if request.method == "POST":
form = TemplateForm(request.POST)
if form.is_valid():
new_tmpl = form.cleaned_data["template"]
try:
inst = job.instance
inst.template.body_template = new_tmpl
job.instance = inst
job.save()
logger.debug(f"Job {rqjob.job_id} template updated")
messages.info(
request,
f"The job {rqjob.job_id} template was updated.",
)
except AttributeError:
logger.debug(
f"Job {rqjob.job_id} template can't be updated.")
messages.warning(
request,
f"The job {rqjob.job_id} template cannot be updated.",
)
else:
messages.warning(request, "Please fix errors below.")
return redirect(link)
def cancel(self, request, object_id):
"""Fetch job and re-try to execute it."""
rqjob = get_object_or_404(RQJob, id=object_id)
logger.debug(f"Cancelling job {rqjob.job_id}...")
link = reverse("admin:autoemails_rqjob_preview", args=[object_id])
# fetch job
try:
job = Job.fetch(rqjob.job_id, connection=scheduler.connection)
logger.debug(f"Job {rqjob.job_id} fetched")
except NoSuchJobError:
logger.debug(f"Job {rqjob.job_id} unavailable")
messages.warning(
request,
"The corresponding job in Redis was probably already executed.",
)
return redirect(link)
if job.is_queued or not job.get_status():
job.cancel() # for "pure" jobs
scheduler.cancel(job) # for scheduler-based jobs
rqjob.status = "cancelled"
rqjob.save()
logger.debug(f"Job {rqjob.job_id} was cancelled.")
messages.info(request, f"The job {rqjob.job_id} was cancelled.")
elif job.is_started:
# Right now we don't know how to test a started job, so we simply
# don't allow such jobs to be cancelled.
logger.debug(
f"Job {rqjob.job_id} has started and cannot be cancelled.")
messages.warning(
request,
f"Job {rqjob.job_id} has started and cannot be cancelled.")
elif job.get_status() not in ("", None):
logger.debug(
f"Job {rqjob.job_id} has unknown status or was already executed."
)
messages.warning(
request, "Job has unknown status or was already executed.")
return redirect(link)
def reschedule(self, request, object_id):
"""Change scheduled execution time to a different timestamp."""
rqjob = get_object_or_404(RQJob, id=object_id)
logger.debug(f"Rescheduling job {rqjob.job_id}...")
link = reverse("admin:autoemails_rqjob_preview", args=[object_id])
# fetch job
try:
job = Job.fetch(rqjob.job_id, connection=scheduler.connection)
logger.debug(f"Job {rqjob.job_id} fetched")
except NoSuchJobError:
logger.debug(f"Job {rqjob.job_id} unavailable")
messages.warning(
request,
"The corresponding job in Redis was probably already executed.",
)
return redirect(link)
if request.method == "POST":
form = RescheduleForm(request.POST)
if form.is_valid():
new_exec = form.cleaned_data["scheduled_execution"]
try:
scheduler.change_execution_time(job, new_exec)
logger.debug(f"Job {rqjob.job_id} rescheduled")
messages.info(
request,
f"The job {rqjob.job_id} was rescheduled to {new_exec}.",
)
except ValueError:
logger.debug(
f"Job {rqjob.job_id} could not be rescheduled.")
messages.warning(
request,
f"The job {rqjob.job_id} was not "
"rescheduled. It is probably already "
"executing or has recently executed.",
)
else:
messages.warning(request, "Please fix errors below.")
return redirect(link)
def reschedule_now(self, request, object_id):
"""Reschedule an existing job so it executes now (+/- refresh time
delta, about 1 minute in default settings)."""
rqjob = get_object_or_404(RQJob, id=object_id)
logger.debug(
f"Executing job {rqjob.job_id} now (scheduling to +- 1min)...")
link = reverse("admin:autoemails_rqjob_preview", args=[object_id])
# fetch job
try:
job = Job.fetch(rqjob.job_id, connection=scheduler.connection)
logger.debug(f"Job {rqjob.job_id} fetched")
except NoSuchJobError:
logger.debug(f"Job {rqjob.job_id} unavailable")
messages.warning(
request,
"The corresponding job in Redis was probably already executed.",
)
return redirect(link)
# new scheduled time: now (in UTC)
now_utc = datetime.utcnow()
try:
scheduler.change_execution_time(job, now_utc)
logger.debug(f"Job {rqjob.job_id} rescheduled to now")
messages.info(request,
f"The job {rqjob.job_id} was rescheduled to now.")
except ValueError:
logger.debug(f"Job {rqjob.job_id} could not be rescheduled.")
messages.warning(
request,
f"The job {rqjob.job_id} was not "
"rescheduled. It is probably already "
"executing or has recently executed.",
)
return redirect(link)
def testActionRemove(self):
trigger = self.trigger
task = self.task
job_ids = MagicMock() # it will mock a QuerySet
# Define a special class inheriting from the mixin we're about to test
# so that we can (indirectly?) test the mixin itself. In some cases
# the mock mechanism will have to be used, because - again - we can
# only indirectly test the behavior of `action_remove()`.
class MockView(ActionManageMixin):
def __init__(self, connection, queue, scheduler, *args, **kwargs):
super().__init__(*args, **kwargs)
self.object = task
self.logger = MagicMock()
self.connection = connection
self.queue = queue
self.scheduler = scheduler
def get_logger(self):
return self.logger
def get_scheduler(self):
return self.scheduler
def get_redis_connection(self):
return self.connection
def get_triggers(self):
objs = [trigger]
triggers = MagicMock()
triggers.__iter__.return_value = iter(objs)
triggers.count.return_value = len(objs)
return triggers
def objects(self):
return dict(task=self.object, event=self.object.event)
@property
def request(self):
# fake request created thanks to RequestFactory from Django
# Test Client
req = RequestFactory().post('/tasks/create')
return req
def get_jobs(self, as_id_list=True):
if not as_id_list:
raise NotImplementedError()
return job_ids
view = MockView(self.connection, self.queue, self.scheduler)
# assertions before the view action is invoked
self.assertEqual(self.scheduler.count(), 0)
self.assertEqual(RQJob.objects.count(), 0)
# view action invoke - it schedules a job
view.action_add(NewInstructorAction)
# additionally enqueue (as opposite to schedule) a blocking job
enqueued_job = self.queue.enqueue(dummy_job)
self.assertTrue(enqueued_job.is_finished)
# ensure both a new Job and a corresponding RQJob were created
self.assertEqual(self.scheduler.count(), 1)
self.assertEqual(RQJob.objects.count(), 1)
# ensure it's the same job
job = next(self.scheduler.get_jobs())
rqjob = RQJob.objects.first()
self.assertEqual(job.get_id(), rqjob.job_id)
# mock a Query Set
# previously enqueued job is added here so that the action_remove
# interface is mocked into removing it from the enqueued jobs
real_job_ids = [job.get_id(), enqueued_job.id]
job_ids.__iter__.return_value = iter(real_job_ids)
job_ids.count.return_value = len(real_job_ids)
# invoke action_remove
view.action_remove(NewInstructorAction)
# ensure there are no scheduled jobs nor RQJob objects
self.assertEqual(self.scheduler.count(), 0)
self.assertEqual(RQJob.objects.count(), 0)
# ensure the previously enqueued job is no longer available
with self.assertRaises(NoSuchJobError):
Job.fetch(enqueued_job.id, connection=self.connection)
# logger.debug is called 6 times (for action_add) and 6 times
# (for action_remove)
self.assertEqual(view.get_logger().debug.call_count, 6 + 6)
def preview(self, request, object_id):
"""Show job + email preview (all details and statuses)."""
rqjob = get_object_or_404(RQJob, id=object_id)
logger.debug(f"Previewing job {rqjob.job_id}...")
try:
job = Job.fetch(rqjob.job_id, connection=scheduler.connection)
job_scheduled = scheduled_execution_time(job.get_id(), scheduler)
instance = job.instance
status = check_status(job)
logger.debug(f"Job {rqjob.job_id} fetched")
# the job may not exist anymore, then we can't retrieve any data
except NoSuchJobError:
job = None
job_scheduled = None
instance = None
status = None
trigger = None
template = None
email = None
adn_context = None
logger.debug(f"Job {rqjob.job_id} unavailable")
# we can try and read properties
else:
try:
trigger = instance.trigger
template = instance.template
email = instance._email()
adn_context = instance.context
except AttributeError:
trigger = None
template = None
email = None
adn_context = None
reschedule_form = None
if job and not job.is_failed:
now_utc = datetime.utcnow() + timedelta(minutes=10)
reschedule_form = RescheduleForm(
initial=dict(scheduled_execution=job_scheduled or now_utc)
)
template_form = None
if instance and job and not job.is_failed:
template_form = TemplateForm(
initial=dict(template=instance.template.body_template)
)
# find prev / next RQJob in the list
previous = RQJob.objects.filter(pk__lt=rqjob.pk).order_by("id").last()
next_ = RQJob.objects.filter(pk__gt=rqjob.pk).order_by("id").first()
context = dict(
self.admin_site.each_context(request),
cl=self.get_changelist_instance(request),
title=f"Preview {rqjob}",
rqjob=rqjob,
job=job,
job_scheduled=job_scheduled,
instance=instance,
status=status,
trigger=trigger,
template=template,
email=email,
adn_context=adn_context,
reschedule_form=reschedule_form,
template_form=template_form,
previous=previous,
next=next_,
)
return TemplateResponse(request, "rqjob_preview.html", context)
