def cancel_all(queue_name):
scheduler = Scheduler()
for job in scheduler.get_jobs():
if job.origin == queue_name:
scheduler.cancel(job)
return dict(status='OK')
def cancel_scheduled_task(request):
job_id = request.GET.get('job_id')
from rq_scheduler import Scheduler
scheduler = Scheduler('parser')
scheduler.cancel(job_id)
return HttpResponse('Success')
def stop_job(task_id, job_id):
logger = g.logger.bind(operation="stop", task_id=task_id, job_id=job_id)
logger.debug("Getting job...")
job = Job.get_by_id(task_id=task_id, job_id=job_id)
if job is None:
logger.error("Job not found in task.")
abort(404)
return
execution = job.get_last_execution()
if execution is not None and execution.status == JobExecution.Status.running:
logger.debug("Stopping current execution...")
executor = current_app.load_executor()
executor.stop_job(job.task, job, execution)
logger.debug("Current execution stopped.")
scheduler = Scheduler("jobs", connection=current_app.redis)
if "enqueued_id" in job.metadata and job.metadata[
"enqueued_id"] in scheduler:
scheduler.cancel(job.metadata["enqueued_id"])
job.scheduled = False
job.save()
logger.debug("Job stopped.")
return get_job_summary(task_id, job_id)
def update_scheduled_connection(connection):
''' schedule a new scrape of the connection source
interval was changed
or the last job was finished and the next needs to be scheduled
'''
repeating_task = connection.schedule
# check to see if schedule is available -- abort if not
# note that ready_to_connect does not verify this
if not repeating_task.interval:
return False
# connect to the rq scheduler
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host'], redis_config['port']
, password=redis_config['password']))
scheduler = Scheduler()
# see if this schedule had a job that was already enqueued
if repeating_task.next_task_id:
# instantiate the job
job = Job(id=repeating_task.next_task_id)
# cancel the old job
scheduler.cancel(job)
# determine how many seconds to wait
delay = _calculate_schedule_delay(repeating_task.interval)
# start a new job
job = scheduler.enqueue_in(datetime.timedelta(seconds=int(delay))
, connect_to_source, connection.id)
# save this id and when it runs next
repeating_task.update(set__next_task_id = job.id)
repeating_task.update(set__next_run_time = (datetime.datetime.utcnow()
+ datetime.timedelta(seconds=delay)))
def cancel_all(queue_name):
scheduler = Scheduler()
for job in scheduler.get_jobs():
if job.origin == queue_name:
scheduler.cancel(job)
return dict(status='OK')
def cancel_scheduled_task(request):
job_id = request.GET.get('job_id')
from rq_scheduler import Scheduler
scheduler = Scheduler('parser')
scheduler.cancel(job_id)
return HttpResponse('Success')
def cancel_scheduled_task(request):
from rq_scheduler import Scheduler
job_id = request.GET.get('job_id')
scheduler = Scheduler('default', connection=tasks.redis_conn)
scheduler.cancel(job_id)
return HttpResponse('Success')
def create_schedules():
from towerdashboard.jobs import refresh_github_branches
scheduler = Scheduler(connection=Redis('redis'))
for j in scheduler.get_jobs():
scheduler.cancel(j)
scheduler.schedule(scheduled_time=datetime.utcnow(),
func=refresh_github_branches,
interval=120, repeat=None, result_ttl=120)
def stop_synchronization(job_name):
r = StrictRedis()
scheduler = Scheduler(connection=r)
if job_name not in scheduler:
logger.error("job not exists")
return
scheduler.cancel(job_name)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("action",
choices=['show', 'remove', 'purge'],
help="Which action to run.")
parser.add_argument(
"object",
help=
"Which object to remove (if removing). Add 'all' where this argument is unused."
)
parser.add_argument(
"-e",
'--env',
choices=['development', 'test', 'production'],
help=
"Run within a specific environment. Otherwise run under the environment defined in the environment variable CS_ENV"
)
args = parser.parse_args()
if args.env != None:
os.environ['CS_ENV'] = args.env
queue_name = os.environ['CS_ENV']
scheduler = Scheduler(queue_name=os.environ['CS_ENV'], connection=Redis())
if (args.action == "show"):
print("\n")
print("=================================")
print(" Job Schedule For {0}".format(os.environ['CS_ENV']))
print("=================================")
print("\n")
for job in scheduler.get_jobs(until=timedelta(hours=24),
with_times=True):
print("ID: {1}\n Job: {0}\n Time: {2}\n".format(
job[0].description, job[0].id, job[1]))
elif (args.action == "remove"):
if (args.object is None):
print("Please specify the job to remove")
else:
jobs = scheduler.get_jobs()
for job in jobs:
if (args.object == job.id):
scheduler.cancel(job.id)
print("Job {0} cancelled from {1}".format(
args.object, os.environ['CS_ENV']))
elif (args.action == "purge"):
count = 0
for job in scheduler.get_jobs():
count += 1
scheduler.cancel(job.id)
print("Purged {0} jobs from {1}".format(count, os.environ['CS_ENV']))
def cancel_job(conn, job_id):
scheduler = Scheduler(connection=conn)
if job_id in scheduler:
scheduler.cancel(job_id)
print " Job: " + job_id + " deleted"
else:
print " Job: " + str(job_id) + " not scheduled"
return None
def register_scheduler():
scheduler = Scheduler('lidarts-bulk', connection=Redis())
list_of_job_instances = scheduler.get_jobs()
for job in list_of_job_instances:
scheduler.cancel(job)
scheduler.schedule(
scheduled_time=datetime.utcnow(),
func='lidarts.tasks.bulk_update_last_seen',
interval=5,
repeat=None,
ttl=10,
)
def kill_schedule(channel, verbose=True):
try:
scheduler = Scheduler(channel, connection=RCONN)
jobs_and_times = scheduler.get_jobs(with_times=True)
for job in jobs_and_times:
print("job", job)
scheduler.cancel(job[0].id)
if verbose:
print("All Jobs Killed")
return True
except Exception as e:
print("Errors in killing jobs", e)
return False
def partial_update(self, request, *args, **kwargs):
print("PERFORM PARTIAL UPDATE !!!!!!!!")
response = super(PeriodicJobViewSet,
self).partial_update(request, *args, **kwargs)
script = self.get_object()
print("------------>", script.interval)
# scheduler add or delete
print("scheduler !!!")
job_id = get_job_id(script.id)
scheduler = Scheduler("default", connection=Redis())
print("job_id", job_id)
job = None
try:
redis = Redis()
job = Job.fetch(job_id, connection=redis)
print("there is a job", job)
scheduler.cancel(job)
print("Cancel the job")
except NoSuchJobError as e:
job = None
finally:
job = None
if job is None and script.interval > 0:
print("Add for schedule ...")
scheduler.schedule(
scheduled_time=datetime.utcnow(
), # Time for first execution, in UTC timezone
func=func3, # Function to be queued
args=[{
"script_id": script.id
}], # Arguments passed into function when executed
id=job_id,
interval=script.
interval, # Time before the function is called again, in seconds
repeat=
None, # Repeat this number of times (None means repeat forever)
result_ttl=script.interval * 2,
)
# do not set a result_ttl value or you set a value larger than the interval
return response
def __remove_from_scheduler(self):
rq_job_ids = self._rq_job_ids
if rq_job_ids:
queue = Queue(SchedulerDB.submit_queue_name, connection=self.con)
scheduler = Scheduler(queue=queue, connection=self.con)
for rq_job_id in rq_job_ids:
try:
scheduler.cancel(rq_job_id)
RqJob.fetch(rq_job_id, connection=self.con).delete()
self.logger.debug(
f"Rq job {rq_job_id} removed from the rq scheduler")
except NoSuchJobError:
pass
self.logger.debug(f"{self} has been removed from the rq scheduler")
def start_scheduler(redis_url, redis_password=None, queue_name='job_scheduler_queue'):
queue = redis_queue(redis_url, redis_password, queue_name)
scheduler = Scheduler(queue_name=queue.name, connection=queue.connection)
queue.empty()
for job in scheduler.get_jobs():
scheduler.cancel(job)
logger.info(f"Removed old job {job} from scheduler.")
# add jobs to scheduler
job = scheduler.cron(
cron_string="* * * * *", # once a minute
func=log_review,
args=[datetime.now(), choice(['Alice', 'Bob', 'Carol', 'Dave'])],
queue_name=queue.name,
repeat=None
)
logger.info(f"Added job {job}")
return scheduler
def test_schedule_jobs(mocker, queue, jobid):
sch = Scheduler(queue=queue, connection=queue.connection)
sch.cancel = mocker.MagicMock()
jobs.schedule_jobs(sch)
assert jobid in sch
assert len(list(sch.get_jobs())) == 1
# running again should have no effect
jobs.schedule_jobs(sch)
assert jobid in sch
assert len(list(sch.get_jobs())) == 1
assert not sch.cancel.called
def perform_stop_job_execution(job, execution, logger, stop_schedule=True):
if execution is None:
if not job.executions:
msg = "No executions found in job."
return (
False,
return_error(msg,
"stop_job_execution",
status=400,
logger=logger),
)
execution = job.get_last_execution()
if execution is not None and execution.status == JobExecution.Status.running:
logger.debug("Stopping current execution...")
executor = current_app.executor
executor.stop_job(job.task, job, execution)
logger.debug("Current execution stopped.")
if "retries" in job.metadata:
job.metadata["retry_count"] = job.metadata["retries"] + 1
job.save()
scheduler = Scheduler("jobs", connection=current_app.redis)
if (stop_schedule and "enqueued_id" in job.metadata
and job.metadata["enqueued_id"] in scheduler):
scheduler.cancel(job.metadata["enqueued_id"])
job.scheduled = False
if execution.error is None:
execution.error = ""
execution.error += "\nUser stopped job execution manually."
execution.status = JobExecution.Status.failed
job.save()
logger.debug("Job stopped.")
return True, None
def setup_scheduler(func, repeat_every=60):
r = get_connection()
scheduled_job_id = r.get(KEY)
scheduler = Scheduler(connection=r)
if scheduled_job_id:
logger.info(f'Canceling old job {scheduled_job_id}')
scheduler.cancel(
scheduled_job_id) # schedule old job before scheduling a new one
job = scheduler.schedule(
scheduled_time=datetime.utcnow(
), # Time for first execution, in UTC timezone
func=func, # Function to be queued
interval=
repeat_every, # Time before the function is called again, in seconds
repeat=None # Repeat this number of times (None means repeat forever)
)
logger.info("Scheduled function %s to be executed every %s seconds" %
(func.__name__, repeat_every))
r.set(KEY, job.id)
def update_scheduled_send(schedule_id):
''' schedule a new sending
interval was changed
or the last job has finished and next needs to be scheduled
'''
schedules = Schedule.objects(id=schedule_id)
if not schedules:
return False
schedule = schedules[0]
# confirm that schedule is valid
if not schedule.interval:
return False
# connect to the rq scheduler
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host'], redis_config['port']
, password=redis_config['password']))
scheduler = Scheduler()
# see if this schedule had a job that was already enqueued
if schedule.next_task_id:
# instantiate the job
job = Job(id=schedule.next_task_id)
# cancel the old job
# tried rescheduling but that was not working
scheduler.cancel(job)
# determine how many seconds to wait
delay = _calculate_schedule_delay(schedule.interval)
# start a new job
job = scheduler.enqueue_in(datetime.timedelta(seconds=int(delay))
, send_scheduled_report, schedule.id)
# save the id of this job and when it next runs
schedule.update(set__next_task_id = job.id)
schedule.update(set__next_run_time = (datetime.datetime.utcnow()
+ datetime.timedelta(seconds=delay)))
def rq_check_job_status_scheduler(job_ids, register_following_job_callback,
redis_url):
jobs = []
status = True
rq_connection = Redis.from_url(redis_url)
try:
for jid in job_ids:
_j = Job.fetch(jid, connection=rq_connection)
jobs.append(_j)
print('Checking job', _j.id)
except NoSuchJobError as e:
status = False
except Exception as e:
status = False
# print(jobs)
try:
if status:
for job in jobs:
if job.status != 'finished':
print('not finished yet')
return
register_following_job_callback(redis_url)
scheduler = Scheduler('high', connection=rq_connection
) # Get a scheduler for the "foo" queue
scheduler.cancel(get_current_job())
else:
raise Exception('One or more downloads failed')
except Exception as e:
print(e)
import traceback
traceback.print_tb(e.__traceback__)
print("Canceling scheduler", get_current_job().id)
scheduler = Scheduler(
'high',
connection=rq_connection) # Get a scheduler for the "foo" queue
scheduler.cancel(get_current_job())
class RqClient:
def __init__(self, conf: RqConfig, prefix: str = ''):
self.redis_conn = Redis(host=conf.HOST, port=conf.PORT, db=conf.DB)
self.queue = Queue(connection=self.redis_conn)
self.prefix = prefix
self.scheduler = Scheduler(connection=self.redis_conn, queue=self.queue)
self.scheduler_conf_path = conf.SCHEDULER_CONF_PATH
self.control = Control(self.redis_conn)
def init_scheduler(self):
# remove old scheduled tasks
for job in self.scheduler.get_jobs():
self.scheduler.cancel(job)
# create new tasks from config file
if self.scheduler_conf_path:
with open(self.scheduler_conf_path) as f:
for entry in json.load(f):
self.scheduler.cron(
entry['schedule'],
f'{self.prefix}.{entry["task"]}',
kwargs=entry['kwargs'] if 'kwargs' in entry else None
)
def send_task(self, name, args=None, time_limit=None, soft_time_limit=None):
try:
job = self.queue.enqueue(f'{self.prefix}.{name}', ttl=time_limit, args=args)
return ResultWrapper(job)
except Exception as ex:
logging.getLogger(__name__).error(ex)
def AsyncResult(self, ident):
try:
return ResultWrapper(Job.fetch(ident, connection=self.redis_conn))
except NoSuchJobError:
logging.getLogger(__name__).warning(f'Job {ident} not found')
return None
class TestScheduler(RQTestCase):
def setUp(self):
super(TestScheduler, self).setUp()
self.scheduler = Scheduler(connection=self.testconn)
def test_birth_and_death_registration(self):
key = Scheduler.scheduler_key
self.assertNotIn(key, self.testconn.keys('*'))
scheduler = Scheduler(connection=self.testconn)
scheduler.register_birth()
self.assertIn(key, self.testconn.keys('*'))
self.assertFalse(self.testconn.hexists(key, 'death'))
self.assertRaises(ValueError, scheduler.register_birth)
scheduler.register_death()
self.assertTrue(self.testconn.hexists(key, 'death'))
def test_create_job(self):
"""
Ensure that jobs are created properly.
"""
job = self.scheduler._create_job(say_hello, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job, job_from_queue)
self.assertEqual(job_from_queue.func, say_hello)
def test_job_not_persisted_if_commit_false(self):
"""
Ensure jobs are only saved to Redis if commit=True.
"""
job = self.scheduler._create_job(say_hello, commit=False)
self.assertEqual(self.testconn.hgetall(job.key), {})
def test_create_scheduled_job(self):
"""
Ensure that scheduled jobs are put in the scheduler queue with the right score
"""
scheduled_time = datetime.now()
job = self.scheduler.enqueue_at(scheduled_time, say_hello)
self.assertEqual(job, Job.fetch(job.id, connection=self.testconn))
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int(scheduled_time.strftime('%s')))
def test_enqueue_in(self):
"""
Ensure that jobs have the right scheduled time.
"""
right_now = datetime.now()
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int((right_now + time_delta).strftime('%s')))
time_delta = timedelta(hours=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int((right_now + time_delta).strftime('%s')))
def test_get_jobs_to_queue(self):
"""
Ensure that jobs scheduled the future are not queued.
"""
now = datetime.now()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertNotIn(job, self.scheduler.get_jobs_to_queue())
def test_enqueue_job(self):
"""
When scheduled job is enqueued, make sure:
- Job is removed from the sorted set of scheduled jobs
- "enqueued_at" attribute is properly set
- Job appears in the right queue
"""
now = datetime.now()
queue_name = 'foo'
scheduler = Scheduler(connection=self.testconn, queue_name=queue_name)
job = scheduler.enqueue_at(now, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(
job, self.testconn.zrange(scheduler.scheduled_jobs_key, 0, 10))
job = Job.fetch(job.id, connection=self.testconn)
self.assertTrue(job.enqueued_at is not None)
queue = scheduler.get_queue_for_job(job)
self.assertIn(job, queue.jobs)
queue = Queue.from_queue_key('rq:queue:{0}'.format(queue_name))
self.assertIn(job, queue.jobs)
def test_cancel_scheduled_job(self):
"""
When scheduled job is canceled, make sure:
- Job is removed from the sorted set of scheduled jobs
"""
# schedule a job to be enqueued one minute from now
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
# cancel the scheduled job and check that it's gone from the set
self.scheduler.cancel(job)
self.assertNotIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
def test_change_execution_time(self):
"""
Ensure ``change_execution_time`` is called, ensure that job's score is updated
"""
job = self.scheduler.enqueue_at(datetime.now(), say_hello)
new_date = datetime(2010, 1, 1)
self.scheduler.change_execution_time(job, new_date)
self.assertEqual(
int(new_date.strftime('%s')),
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
def test_args_kwargs_are_passed_correctly(self):
"""
Ensure that arguments and keyword arguments are properly saved to jobs.
"""
job = self.scheduler.enqueue_at(datetime.now(), simple_addition, 1, 1,
1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_at(datetime.now(),
simple_addition,
z=1,
y=1,
x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_at(datetime.now(),
simple_addition,
1,
z=1,
y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1, ))
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_in(time_delta,
simple_addition,
z=1,
y=1,
x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_in(time_delta,
simple_addition,
1,
z=1,
y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1, ))
def test_interval_and_repeat_persisted_correctly(self):
"""
Ensure that interval and repeat attributes get correctly saved in Redis.
"""
job = self.scheduler.enqueue(datetime.now(),
say_hello,
interval=10,
repeat=11)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(int(job_from_queue.interval), 10)
self.assertEqual(int(job_from_queue.repeat), 11)
def test_repeat_without_interval_raises_error(self):
# Ensure that an error is raised if repeat is specified without interval
def create_job():
self.scheduler.enqueue(datetime.now(), say_hello, repeat=11)
self.assertRaises(ValueError, create_job)
def test_job_with_intervals_get_rescheduled(self):
"""
Ensure jobs with interval attribute are put back in the scheduler
"""
time_now = datetime.now()
interval = 10
job = self.scheduler.enqueue(time_now, say_hello, interval=interval)
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int(time_now.strftime('%s')) + interval)
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, None,
say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int(time_now.strftime('%s')) + interval)
def test_job_with_repeat(self):
"""
Ensure jobs with repeat attribute are put back in the scheduler
X (repeat) number of times
"""
time_now = datetime.now()
interval = 10
# If job is repeated once, the job shouldn't be put back in the queue
job = self.scheduler.enqueue(time_now,
say_hello,
interval=interval,
repeat=1)
self.scheduler.enqueue_job(job)
self.assertNotIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.enqueue(time_now,
say_hello,
interval=interval,
repeat=2)
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.scheduler.enqueue_job(job)
self.assertNotIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
time_now = datetime.now()
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, 1, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.enqueue_periodic(time_now, interval, 2, say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.scheduler.enqueue_job(job)
self.assertNotIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
def test_missing_jobs_removed_from_scheduler(self):
"""
Ensure jobs that don't exist when queued are removed from the scheduler.
"""
job = self.scheduler.enqueue(datetime.now(), say_hello)
job.cancel()
self.scheduler.get_jobs_to_queue()
self.assertNotIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
def test_periodic_jobs_sets_ttl(self):
"""
Ensure periodic jobs set result_ttl to infinite.
"""
job = self.scheduler.enqueue(datetime.now(), say_hello, interval=5)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.result_ttl, -1)
def cancel_job_view(job_id):
scheduler = Scheduler()
scheduler.cancel(job_id)
return dict(status='OK')
class TestScheduler(RQTestCase):
def setUp(self):
super(TestScheduler, self).setUp()
self.scheduler = Scheduler(connection=self.testconn)
def test_birth_and_death_registration(self):
"""
When scheduler registers it's birth, besides creating a key, it should
also set an expiry that's a few seconds longer than it's polling
interval so it automatically expires if scheduler is unexpectedly
terminated.
"""
key = Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=20)
scheduler.register_birth()
self.assertIn(key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(key), 30)
self.assertFalse(self.testconn.hexists(key, 'death'))
self.assertRaises(ValueError, scheduler.register_birth)
scheduler.register_death()
self.assertTrue(self.testconn.hexists(key, 'death'))
def test_create_job(self):
"""
Ensure that jobs are created properly.
"""
job = self.scheduler._create_job(say_hello, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job, job_from_queue)
self.assertEqual(job_from_queue.func, say_hello)
def test_job_not_persisted_if_commit_false(self):
"""
Ensure jobs are only saved to Redis if commit=True.
"""
job = self.scheduler._create_job(say_hello, commit=False)
self.assertEqual(self.testconn.hgetall(job.key), {})
def test_create_scheduled_job(self):
"""
Ensure that scheduled jobs are put in the scheduler queue with the right score
"""
scheduled_time = datetime.utcnow()
job = self.scheduler.enqueue_at(scheduled_time, say_hello)
self.assertEqual(job, Job.fetch(job.id, connection=self.testconn))
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(scheduled_time))
def test_enqueue_in(self):
"""
Ensure that jobs have the right scheduled time.
"""
right_now = datetime.utcnow()
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(right_now + time_delta))
time_delta = timedelta(hours=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(right_now + time_delta))
def test_get_jobs(self):
"""
Ensure get_jobs() returns all jobs until the specified time.
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs(now))
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertIn(job, self.scheduler.get_jobs(timedelta(hours=1, seconds=1)))
self.assertIn(job, [j[0] for j in self.scheduler.get_jobs(with_times=True)])
self.assertIsInstance(self.scheduler.get_jobs(with_times=True)[0][1], datetime)
self.assertNotIn(job, self.scheduler.get_jobs(timedelta(minutes=59, seconds=59)))
def test_get_jobs_to_queue(self):
"""
Ensure that jobs scheduled the future are not queued.
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertNotIn(job, self.scheduler.get_jobs_to_queue())
def test_enqueue_job(self):
"""
When scheduled job is enqueued, make sure:
- Job is removed from the sorted set of scheduled jobs
- "enqueued_at" attribute is properly set
- Job appears in the right queue
"""
now = datetime.utcnow()
queue_name = 'foo'
scheduler = Scheduler(connection=self.testconn, queue_name=queue_name)
job = scheduler.enqueue_at(now, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(job, tl(self.testconn.zrange(scheduler.scheduled_jobs_key, 0, 10)))
job = Job.fetch(job.id, connection=self.testconn)
self.assertTrue(job.enqueued_at is not None)
queue = scheduler.get_queue_for_job(job)
self.assertIn(job, queue.jobs)
queue = Queue.from_queue_key('rq:queue:{0}'.format(queue_name))
self.assertIn(job, queue.jobs)
def test_job_membership(self):
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler)
self.assertIn(job.id, self.scheduler)
self.assertNotIn("non-existing-job-id", self.scheduler)
def test_cancel_scheduled_job(self):
"""
When scheduled job is canceled, make sure:
- Job is removed from the sorted set of scheduled jobs
"""
# schedule a job to be enqueued one minute from now
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
# cancel the scheduled job and check that it's gone from the set
self.scheduler.cancel(job)
self.assertNotIn(job.id, tl(self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1)))
def test_change_execution_time(self):
"""
Ensure ``change_execution_time`` is called, ensure that job's score is updated
"""
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello)
new_date = datetime(2010, 1, 1)
self.scheduler.change_execution_time(job, new_date)
self.assertEqual(to_unix(new_date),
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
self.scheduler.cancel(job)
self.assertRaises(ValueError, self.scheduler.change_execution_time, job, new_date)
def test_args_kwargs_are_passed_correctly(self):
"""
Ensure that arguments and keyword arguments are properly saved to jobs.
"""
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, z=1, y=1, x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, 1, z=1, y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1,))
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_in(time_delta, simple_addition, z=1, y=1, x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, z=1, y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1,))
def test_enqueue_is_deprecated(self):
"""
Ensure .enqueue() throws a DeprecationWarning
"""
with warnings.catch_warnings(record=True) as w:
# Enable all warnings
warnings.simplefilter("always")
job = self.scheduler.enqueue(datetime.utcnow(), say_hello)
self.assertEqual(1, len(w))
self.assertEqual(w[0].category, DeprecationWarning)
def test_enqueue_periodic(self):
"""
Ensure .enqueue_periodic() throws a DeprecationWarning
"""
with warnings.catch_warnings(record=True) as w:
# Enable all warnings
warnings.simplefilter("always")
job = self.scheduler.enqueue_periodic(datetime.utcnow(), 1, None, say_hello)
self.assertEqual(1, len(w))
self.assertEqual(w[0].category, DeprecationWarning)
def test_interval_and_repeat_persisted_correctly(self):
"""
Ensure that interval and repeat attributes get correctly saved in Redis.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=10, repeat=11)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.meta['interval'], 10)
self.assertEqual(job_from_queue.meta['repeat'], 11)
def test_repeat_without_interval_raises_error(self):
# Ensure that an error is raised if repeat is specified without interval
def create_job():
self.scheduler.schedule(datetime.utcnow(), say_hello, repeat=11)
self.assertRaises(ValueError, create_job)
def test_job_with_intervals_get_rescheduled(self):
"""
Ensure jobs with interval attribute are put back in the scheduler
"""
time_now = datetime.utcnow()
interval = 10
job = self.scheduler.schedule(time_now, say_hello, interval=interval)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(time_now) + interval)
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, None, say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(time_now) + interval)
def test_job_with_repeat(self):
"""
Ensure jobs with repeat attribute are put back in the scheduler
X (repeat) number of times
"""
time_now = datetime.utcnow()
interval = 10
# If job is repeated once, the job shouldn't be put back in the queue
job = self.scheduler.schedule(time_now, say_hello, interval=interval, repeat=1)
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.schedule(time_now, say_hello, interval=interval, repeat=2)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
time_now = datetime.utcnow()
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, 1, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.enqueue_periodic(time_now, interval, 2, say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
def test_missing_jobs_removed_from_scheduler(self):
"""
Ensure jobs that don't exist when queued are removed from the scheduler.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello)
job.cancel()
self.scheduler.get_jobs_to_queue()
self.assertNotIn(job.id, tl(self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1)))
def test_periodic_jobs_sets_ttl(self):
"""
Ensure periodic jobs set result_ttl to infinite.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.result_ttl, -1)
def test_run(self):
"""
Check correct signal handling in Scheduler.run().
"""
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, self.scheduler.run)
thread.join()
def test_scheduler_w_o_explicit_connection(self):
"""
Ensure instantiating Scheduler w/o explicit connection works.
"""
s = Scheduler()
self.assertEqual(s.connection, self.testconn)
def test_no_functions_from__main__module(self):
"""
Ensure functions from the __main__ module are not accepted for scheduling.
"""
def dummy():
return 1
# Fake __main__ module function
dummy.__module__ = "__main__"
self.assertRaises(ValueError, self.scheduler._create_job, dummy)
class RqClient:
def __init__(self, conf: RqConfig, prefix: str = ''):
self.redis_conn = Redis(host=conf.HOST, port=conf.PORT, db=conf.DB)
self.queue = Queue(connection=self.redis_conn)
self.prefix = prefix
self.scheduler = Scheduler(connection=self.redis_conn,
queue=self.queue)
self.scheduler_conf_path = conf.SCHEDULER_CONF_PATH
self.control = Control(self.redis_conn)
def init_scheduler(self):
# remove old scheduled tasks
for job in self.scheduler.get_jobs():
self.scheduler.cancel(job)
# create new tasks from config file
if self.scheduler_conf_path:
with open(self.scheduler_conf_path) as f:
for entry in json.load(f):
self.scheduler.cron(
entry['schedule'],
f'{self.prefix}.{entry["task"]}',
kwargs=entry['kwargs'] if 'kwargs' in entry else None)
@staticmethod
def _resolve_limit(softl, hardl):
if softl is not None and hardl is not None:
return min(softl, hardl)
elif softl is not None:
return softl
elif hardl is not None:
return hardl
return None
def send_task(self,
name,
args=None,
time_limit=None,
soft_time_limit=None):
"""
Send a task to the worker.
Please note that Rq does not know hard vs. soft time limit. In case both
values are filled in (time_limit, soft_time_limit), the smaller one is
selected. Otherwise, the non-None is applied.
"""
time_limit = self._resolve_limit(time_limit, soft_time_limit)
try:
job = self.queue.enqueue(f'{self.prefix}.{name}',
job_timeoutx=time_limit,
args=args)
return ResultWrapper(job)
except Exception as ex:
logging.getLogger(__name__).error(ex)
def AsyncResult(self, ident):
try:
return ResultWrapper(Job.fetch(ident, connection=self.redis_conn))
except NoSuchJobError:
logging.getLogger(__name__).warning(f'Job {ident} not found')
return None
def schedule_delete(schedule_inst):
scheduler = Scheduler(connection=Redis()) # Get a scheduler for the "default" queue
scheduler.cancel(schedule_inst)
return "Deleted"
class RqClient(AbstractBgClient):
def __init__(self, conf: RqConfig, prefix: str = ''):
self.redis_conn = Redis(host=conf.HOST, port=conf.PORT, db=conf.DB)
self.queue = Queue(connection=self.redis_conn)
self.prefix = prefix
self.scheduler = Scheduler(connection=self.redis_conn,
queue=self.queue)
self.scheduler_conf_path = conf.SCHEDULER_CONF_PATH
self._control = Control(self.redis_conn)
def init_scheduler(self):
# remove old scheduled tasks
for job in self.scheduler.get_jobs():
self.scheduler.cancel(job)
# create new tasks from config file
if self.scheduler_conf_path:
with open(self.scheduler_conf_path) as f:
for entry in json.load(f):
self.scheduler.cron(
entry['schedule'],
f'{self.prefix}.{entry["task"]}',
kwargs=entry['kwargs'] if 'kwargs' in entry else None,
use_local_timezone=True,
)
logging.getLogger(__name__).info(
f'Loaded configuration for Rq-scheduler from {self.scheduler_conf_path}'
)
else:
logging.getLogger(__name__).warning(
'No Rq-scheduler configuration path defined. '
'Regular system maintenance will be disabled which may lead to disks becoming full.'
)
@staticmethod
def _resolve_limit(softl, hardl):
if softl is not None and hardl is not None:
return min(softl, hardl)
elif softl is not None:
return softl
elif hardl is not None:
return hardl
return None
@property
def control(self):
return self._control
def send_task(self,
name,
ans_type: Type[T],
args=None,
time_limit=None,
soft_time_limit=None) -> ResultWrapper[T]:
"""
Send a task to the worker.
Please note that Rq does not know hard vs. soft time limit. In case both
values are filled in (time_limit, soft_time_limit), the smaller one is
selected. Otherwise, the non-None is applied.
"""
time_limit = self._resolve_limit(time_limit, soft_time_limit)
try:
job = self.queue.enqueue(f'{self.prefix}.{name}',
job_timeout=time_limit,
args=args)
return ResultWrapper(job)
except Exception as ex:
logging.getLogger(__name__).error(ex)
def get_task_error(self, task_id):
try:
job = Job.fetch(task_id, connection=self.redis_conn)
if job.get_status() == 'failed':
return BgCalcError(job.exc_info)
except NoSuchJobError as ex:
return CalcTaskNotFoundError(ex)
return None
def AsyncResult(self, ident):
try:
return ResultWrapper(Job.fetch(ident, connection=self.redis_conn))
except NoSuchJobError:
logging.getLogger(__name__).warning(f'Job {ident} not found')
return None
def is_wrapped_user_error(self, err):
return isinstance(err, UserActionException)
class TestScheduler(RQTestCase):
def setUp(self):
super(TestScheduler, self).setUp()
self.scheduler = Scheduler(connection=self.testconn)
def test_acquire_lock(self):
"""
When scheduler acquires a lock, besides creating a key, it should
also set an expiry that's a few seconds longer than it's polling
interval so it automatically expires if scheduler is unexpectedly
terminated.
"""
key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=20)
self.assertTrue(scheduler.acquire_lock())
self.assertIn(key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(key), 30)
scheduler.remove_lock()
self.assertNotIn(key, tl(self.testconn.keys('*')))
def test_no_two_schedulers_acquire_lock(self):
"""
Ensure that no two schedulers can acquire the lock at the
same time. When removing the lock, only the scheduler which
originally acquired the lock can remove the lock.
"""
key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler1 = Scheduler(connection=self.testconn, interval=20)
scheduler2 = Scheduler(connection=self.testconn, interval=20)
self.assertTrue(scheduler1.acquire_lock())
self.assertFalse(scheduler2.acquire_lock())
self.assertIn(key, tl(self.testconn.keys('*')))
scheduler2.remove_lock()
self.assertIn(key, tl(self.testconn.keys('*')))
scheduler1.remove_lock()
self.assertNotIn(key, tl(self.testconn.keys('*')))
def test_create_job(self):
"""
Ensure that jobs are created properly.
"""
job = self.scheduler._create_job(say_hello, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job, job_from_queue)
self.assertEqual(job_from_queue.func, say_hello)
def test_create_job_with_ttl(self):
"""
Ensure that TTL is passed to RQ.
"""
job = self.scheduler._create_job(say_hello, ttl=2, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(2, job_from_queue.ttl)
def test_create_job_with_id(self):
"""
Ensure that ID is passed to RQ.
"""
job = self.scheduler._create_job(say_hello,
id='id test',
args=(),
kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('id test', job_from_queue.id)
def test_create_job_with_description(self):
"""
Ensure that description is passed to RQ.
"""
job = self.scheduler._create_job(say_hello,
description='description',
args=(),
kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('description', job_from_queue.description)
def test_create_job_with_timeout(self):
"""
Ensure that timeout is passed to RQ.
"""
timeout = 13
job = self.scheduler._create_job(say_hello,
timeout=13,
args=(),
kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(timeout, job_from_queue.timeout)
def test_job_not_persisted_if_commit_false(self):
"""
Ensure jobs are only saved to Redis if commit=True.
"""
job = self.scheduler._create_job(say_hello, commit=False)
self.assertEqual(self.testconn.hgetall(job.key), {})
def test_create_scheduled_job(self):
"""
Ensure that scheduled jobs are put in the scheduler queue with the right score
"""
scheduled_time = datetime.utcnow()
job = self.scheduler.enqueue_at(scheduled_time, say_hello)
self.assertEqual(job, Job.fetch(job.id, connection=self.testconn))
self.assertIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(scheduled_time))
def test_create_job_with_meta(self):
"""
Ensure that meta information on the job is passed to rq
"""
expected = {'say': 'hello'}
job = self.scheduler._create_job(say_hello, meta=expected)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(expected, job_from_queue.meta)
def test_enqueue_at_sets_timeout(self):
"""
Ensure that a job scheduled via enqueue_at can be created with
a custom timeout.
"""
timeout = 13
job = self.scheduler.enqueue_at(datetime.utcnow(),
say_hello,
timeout=timeout)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.timeout, timeout)
def test_enqueue_at_sets_job_id(self):
"""
Ensure that a job scheduled via enqueue_at can be created with
a custom job id.
"""
job_id = 'test_id'
job = self.scheduler.enqueue_at(datetime.utcnow(),
say_hello,
job_id=job_id)
self.assertEqual(job.id, job_id)
def test_enqueue_at_sets_job_ttl(self):
"""
Ensure that a job scheduled via enqueue_at can be created with a custom job ttl.
"""
job_ttl = 123456789
job = self.scheduler.enqueue_at(datetime.utcnow(),
say_hello,
job_ttl=job_ttl)
self.assertEqual(job.ttl, job_ttl)
def test_enqueue_at_sets_job_result_ttl(self):
"""
Ensure that a job scheduled via enqueue_at can be created with a custom result ttl.
"""
job_result_ttl = 1234567890
job = self.scheduler.enqueue_at(datetime.utcnow(),
say_hello,
job_result_ttl=job_result_ttl)
self.assertEqual(job.result_ttl, job_result_ttl)
def test_enqueue_at_sets_meta(self):
"""
Ensure that a job scheduled via enqueue_at can be created with a custom meta.
"""
meta = {'say': 'hello'}
job = self.scheduler.enqueue_at(datetime.utcnow(),
say_hello,
meta=meta)
self.assertEqual(job.meta, meta)
def test_enqueue_in(self):
"""
Ensure that jobs have the right scheduled time.
"""
right_now = datetime.utcnow()
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(right_now + time_delta))
time_delta = timedelta(hours=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(right_now + time_delta))
def test_enqueue_in_sets_timeout(self):
"""
Ensure that a job scheduled via enqueue_in can be created with
a custom timeout.
"""
timeout = 13
job = self.scheduler.enqueue_in(timedelta(minutes=1),
say_hello,
timeout=timeout)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.timeout, timeout)
def test_enqueue_in_sets_job_id(self):
"""
Ensure that a job scheduled via enqueue_in can be created with
a custom job id.
"""
job_id = 'test_id'
job = self.scheduler.enqueue_in(timedelta(minutes=1),
say_hello,
job_id=job_id)
self.assertEqual(job.id, job_id)
def test_enqueue_in_sets_job_ttl(self):
"""
Ensure that a job scheduled via enqueue_in can be created with a custom job ttl.
"""
job_ttl = 123456789
job = self.scheduler.enqueue_in(timedelta(minutes=1),
say_hello,
job_ttl=job_ttl)
self.assertEqual(job.ttl, job_ttl)
def test_enqueue_in_sets_job_result_ttl(self):
"""
Ensure that a job scheduled via enqueue_in can be created with a custom result ttl.
"""
job_result_ttl = 1234567890
job = self.scheduler.enqueue_in(timedelta(minutes=1),
say_hello,
job_result_ttl=job_result_ttl)
self.assertEqual(job.result_ttl, job_result_ttl)
def test_enqueue_in_sets_meta(self):
"""
Ensure that a job scheduled via enqueue_in sets meta.
"""
meta = {'say': 'hello'}
job = self.scheduler.enqueue_in(timedelta(minutes=1),
say_hello,
meta=meta)
self.assertEqual(job.meta, meta)
def test_count(self):
now = datetime.utcnow()
self.scheduler.enqueue_at(now, say_hello)
self.assertEqual(self.scheduler.count(), 1)
future_time = now + timedelta(hours=1)
future_test_time = now + timedelta(minutes=59, seconds=59)
self.scheduler.enqueue_at(future_time, say_hello)
self.assertEqual(
self.scheduler.count(timedelta(minutes=59, seconds=59)), 1)
self.assertEqual(self.scheduler.count(future_test_time), 1)
self.assertEqual(self.scheduler.count(), 2)
def test_get_jobs(self):
"""
Ensure get_jobs() returns all jobs until the specified time.
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs(now))
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertIn(job,
self.scheduler.get_jobs(timedelta(hours=1, seconds=1)))
self.assertIn(job,
[j[0] for j in self.scheduler.get_jobs(with_times=True)])
self.assertIsInstance(
list(self.scheduler.get_jobs(with_times=True))[0][1], datetime)
self.assertNotIn(
job, self.scheduler.get_jobs(timedelta(minutes=59, seconds=59)))
def test_get_jobs_slice(self):
"""
Ensure get_jobs() returns the appropriate slice of all jobs using offset and length.
"""
now = datetime.utcnow()
future_time = now + timedelta(hours=1)
future_test_time = now + timedelta(minutes=59, seconds=59)
# Schedule each job a second later than the previous job,
# otherwise Redis will return jobs that have the same scheduled time in
# lexicographical order (not the order in which we enqueued them)
now_jobs = [
self.scheduler.enqueue_at(now + timedelta(seconds=x), say_hello)
for x in range(15)
]
future_jobs = [
self.scheduler.enqueue_at(future_time + timedelta(seconds=x),
say_hello) for x in range(15)
]
expected_slice = now_jobs[
5:] + future_jobs[:
10] # last 10 from now_jobs and first 10 from future_jobs
expected_until_slice = now_jobs[5:] # last 10 from now_jobs
jobs = self.scheduler.get_jobs()
jobs_slice = self.scheduler.get_jobs(offset=5, length=20)
jobs_until_slice = self.scheduler.get_jobs(future_test_time,
offset=5,
length=20)
self.assertEqual(now_jobs + future_jobs, list(jobs))
self.assertEqual(expected_slice, list(jobs_slice))
self.assertEqual(expected_until_slice, list(jobs_until_slice))
def test_get_jobs_to_queue(self):
"""
Ensure that jobs scheduled the future are not queued.
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertNotIn(job, self.scheduler.get_jobs_to_queue())
def test_enqueue_job(self):
"""
When scheduled job is enqueued, make sure:
- Job is removed from the sorted set of scheduled jobs
- "enqueued_at" attribute is properly set
- Job appears in the right queue
- Queue is recognized by rq's Queue.all()
"""
now = datetime.utcnow()
queue_name = 'foo'
scheduler = Scheduler(connection=self.testconn, queue_name=queue_name)
job = scheduler.enqueue_at(now, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(
job, tl(self.testconn.zrange(scheduler.scheduled_jobs_key, 0, 10)))
job = Job.fetch(job.id, connection=self.testconn)
self.assertTrue(job.enqueued_at is not None)
queue = scheduler.get_queue_for_job(job)
self.assertIn(job, queue.jobs)
queue = Queue.from_queue_key('rq:queue:{0}'.format(queue_name))
self.assertIn(job, queue.jobs)
self.assertIn(queue, Queue.all())
def test_enqueue_job_with_scheduler_queue(self):
"""
Ensure that job is enqueued correctly when the scheduler is bound
to a queue object and job queue name is not provided.
"""
queue = Queue('foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello)
scheduler_queue = scheduler.get_queue_for_job(job)
self.assertEqual(queue, scheduler_queue)
scheduler.enqueue_job(job)
self.assertTrue(job.enqueued_at is not None)
self.assertIn(job, queue.jobs)
self.assertIn(queue, Queue.all())
def test_enqueue_job_with_job_queue_name(self):
"""
Ensure that job is enqueued correctly when queue_name is provided
at job creation
"""
queue = Queue('foo', connection=self.testconn)
job_queue = Queue('job_foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello, queue_name='job_foo')
self.assertEqual(scheduler.get_queue_for_job(job), job_queue)
scheduler.enqueue_job(job)
self.assertTrue(job.enqueued_at is not None)
self.assertIn(job, job_queue.jobs)
self.assertIn(job_queue, Queue.all())
def test_enqueue_at_with_job_queue_name(self):
"""
Ensure that job is enqueued correctly when queue_name is provided
to enqueue_at
"""
queue = Queue('foo', connection=self.testconn)
job_queue = Queue('job_foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler.enqueue_at(datetime.utcnow(),
say_hello,
queue_name='job_foo')
self.assertEqual(scheduler.get_queue_for_job(job), job_queue)
self.scheduler.enqueue_job(job)
self.assertTrue(job.enqueued_at is not None)
self.assertIn(job, job_queue.jobs)
self.assertIn(job_queue, Queue.all())
def test_job_membership(self):
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler)
self.assertIn(job.id, self.scheduler)
self.assertNotIn("non-existing-job-id", self.scheduler)
def test_cancel_scheduled_job(self):
"""
When scheduled job is canceled, make sure:
- Job is removed from the sorted set of scheduled jobs
"""
# schedule a job to be enqueued one minute from now
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
# cancel the scheduled job and check that it's gone from the set
self.scheduler.cancel(job)
self.assertNotIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
def test_change_execution_time(self):
"""
Ensure ``change_execution_time`` is called, ensure that job's score is updated
"""
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello)
new_date = datetime(2010, 1, 1)
self.scheduler.change_execution_time(job, new_date)
self.assertEqual(
to_unix(new_date),
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
self.scheduler.cancel(job)
self.assertRaises(ValueError, self.scheduler.change_execution_time,
job, new_date)
def test_args_kwargs_are_passed_correctly(self):
"""
Ensure that arguments and keyword arguments are properly saved to jobs.
"""
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, 1,
1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_at(datetime.utcnow(),
simple_addition,
z=1,
y=1,
x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_at(datetime.utcnow(),
simple_addition,
1,
z=1,
y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1, ))
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_in(time_delta,
simple_addition,
z=1,
y=1,
x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_in(time_delta,
simple_addition,
1,
z=1,
y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1, ))
def test_interval_and_repeat_persisted_correctly(self):
"""
Ensure that interval and repeat attributes are correctly saved.
"""
job = self.scheduler.schedule(datetime.utcnow(),
say_hello,
interval=10,
repeat=11)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.meta['interval'], 10)
self.assertEqual(job_from_queue.meta['repeat'], 11)
def test_crontab_persisted_correctly(self):
"""
Ensure that crontab attribute gets correctly saved in Redis.
"""
# create a job that runs one minute past each whole hour
job = self.scheduler.cron("1 * * * *", say_hello)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.meta['cron_string'], "1 * * * *")
# get the scheduled_time and convert it to a datetime object
unix_time = self.testconn.zscore(self.scheduler.scheduled_jobs_key,
job.id)
datetime_time = from_unix(unix_time)
# check that minute=1, seconds=0, and is within an hour
assert datetime_time.minute == 1
assert datetime_time.second == 0
assert datetime_time - datetime.utcnow() < timedelta(hours=1)
def test_crontab_persisted_correctly_with_local_timezone(self):
"""
Ensure that crontab attribute gets correctly saved in Redis when using local TZ.
"""
# create a job that runs one minute past each whole hour
job = self.scheduler.cron("0 15 * * *",
say_hello,
use_local_timezone=True)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.meta['cron_string'], "0 15 * * *")
# get the scheduled_time and convert it to a datetime object
unix_time = self.testconn.zscore(self.scheduler.scheduled_jobs_key,
job.id)
datetime_time = from_unix(unix_time)
expected_datetime_in_local_tz = datetime.now(
get_utc_timezone()).replace(hour=15,
minute=0,
second=0,
microsecond=0)
assert datetime_time.time(
) == expected_datetime_in_local_tz.astimezone(
get_utc_timezone()).time()
def test_crontab_rescheduled_correctly_with_local_timezone(self):
# Create a job with a cronjob_string
job = self.scheduler.cron("1 15 * * *",
say_hello,
use_local_timezone=True)
# change crontab
job.meta['cron_string'] = "2 15 * * *"
# reenqueue the job
self.scheduler.enqueue_job(job)
# get the scheduled_time and convert it to a datetime object
unix_time = self.testconn.zscore(self.scheduler.scheduled_jobs_key,
job.id)
datetime_time = from_unix(unix_time)
expected_datetime_in_local_tz = datetime.now(
get_utc_timezone()).replace(hour=15,
minute=2,
second=0,
microsecond=0)
assert datetime_time.time(
) == expected_datetime_in_local_tz.astimezone(
get_utc_timezone()).time()
def test_crontab_sets_timeout(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom timeout.
"""
timeout = 13
job = self.scheduler.cron("1 * * * *", say_hello, timeout=timeout)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.timeout, timeout)
def test_crontab_sets_id(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom id
"""
job_id = "hello-job-id"
job = self.scheduler.cron("1 * * * *", say_hello, id=job_id)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_id, job_from_queue.id)
def test_crontab_sets_default_result_ttl(self):
"""
Ensure that a job scheduled via crontab gets proper default
result_ttl (-1) periodic tasks.
"""
job = self.scheduler.cron("1 * * * *", say_hello)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(-1, job_from_queue.result_ttl)
def test_crontab_sets_description(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom description
"""
description = 'test description'
job = self.scheduler.cron("1 * * * *",
say_hello,
description=description)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(description, job_from_queue.description)
def test_repeat_without_interval_raises_error(self):
# Ensure that an error is raised if repeat is specified without interval
def create_job():
self.scheduler.schedule(datetime.utcnow(), say_hello, repeat=11)
self.assertRaises(ValueError, create_job)
def test_job_with_intervals_get_rescheduled(self):
"""
Ensure jobs with interval attribute are put back in the scheduler
"""
time_now = datetime.utcnow()
interval = 10
job = self.scheduler.schedule(time_now, say_hello, interval=interval)
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(time_now) + interval)
def test_job_with_interval_can_set_meta(self):
"""
Ensure that jobs with interval attribute can be created with meta
"""
time_now = datetime.utcnow()
interval = 10
meta = {'say': 'hello'}
job = self.scheduler.schedule(time_now,
say_hello,
interval=interval,
meta=meta)
self.scheduler.enqueue_job(job)
self.assertEqual(job.meta, meta)
def test_job_with_crontab_get_rescheduled(self):
# Create a job with a cronjob_string
job = self.scheduler.cron("1 * * * *", say_hello)
# current unix_time
old_next_scheduled_time = self.testconn.zscore(
self.scheduler.scheduled_jobs_key, job.id)
# change crontab
job.meta['cron_string'] = "2 * * * *"
# enqueue the job
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# check that next scheduled time has changed
self.assertNotEqual(
old_next_scheduled_time,
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
# check that new next scheduled time is set correctly
expected_next_scheduled_time = to_unix(
get_next_scheduled_time("2 * * * *"))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
expected_next_scheduled_time)
def test_job_with_repeat(self):
"""
Ensure jobs with repeat attribute are put back in the scheduler
X (repeat) number of times
"""
time_now = datetime.utcnow()
interval = 10
# If job is repeated once, the job shouldn't be put back in the queue
job = self.scheduler.schedule(time_now,
say_hello,
interval=interval,
repeat=1)
self.scheduler.enqueue_job(job)
self.assertNotIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.schedule(time_now,
say_hello,
interval=interval,
repeat=2)
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.scheduler.enqueue_job(job)
self.assertNotIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
def test_missing_jobs_removed_from_scheduler(self):
"""
Ensure jobs that don't exist when queued are removed from the scheduler.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello)
job.cancel()
list(self.scheduler.get_jobs_to_queue())
self.assertIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
job.delete()
list(self.scheduler.get_jobs_to_queue())
self.assertNotIn(
job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
def test_periodic_jobs_sets_result_ttl(self):
"""
Ensure periodic jobs set result_ttl to infinite.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.result_ttl, -1)
def test_periodic_jobs_sets_ttl(self):
"""
Ensure periodic jobs sets correctly ttl.
"""
job = self.scheduler.schedule(datetime.utcnow(),
say_hello,
interval=5,
ttl=4)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.ttl, 4)
def test_periodic_jobs_sets_meta(self):
"""
Ensure periodic jobs sets correctly meta.
"""
meta = {'say': 'hello'}
job = self.scheduler.schedule(datetime.utcnow(),
say_hello,
interval=5,
meta=meta)
self.assertEqual(meta, job.meta)
def test_periodic_job_sets_id(self):
"""
Ensure that ID is passed to RQ by schedule.
"""
job = self.scheduler.schedule(datetime.utcnow(),
say_hello,
interval=5,
id='id test')
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('id test', job.id)
def test_periodic_job_sets_description(self):
"""
Ensure that description is passed to RQ by schedule.
"""
job = self.scheduler.schedule(datetime.utcnow(),
say_hello,
interval=5,
description='description')
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('description', job.description)
def test_run(self):
"""
Check correct signal handling in Scheduler.run().
"""
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, self.scheduler.run)
thread.join()
def test_run_burst(self):
"""
Check burst mode of Scheduler.run().
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
self.assertEqual(len(list(self.scheduler.get_jobs())), 1)
self.scheduler.run(burst=True)
self.assertEqual(len(list(self.scheduler.get_jobs())), 0)
def test_scheduler_w_o_explicit_connection(self):
"""
Ensure instantiating Scheduler w/o explicit connection works.
"""
s = Scheduler()
self.assertEqual(s.connection, self.testconn)
def test_small_float_interval(self):
"""
Test that scheduler accepts 'interval' of type float, less than 1 second.
"""
key = Scheduler.scheduler_key
lock_key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn,
interval=0.1) # testing interval = 0.1 second
self.assertEqual(scheduler._interval, 0.1)
#acquire lock
self.assertTrue(scheduler.acquire_lock())
self.assertIn(lock_key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(lock_key), 10) # int(0.1) + 10 = 10
#enqueue a job
now = datetime.utcnow()
job = scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
self.assertEqual(len(list(self.scheduler.get_jobs())), 1)
#remove the lock
scheduler.remove_lock()
#test that run works with the small floating-point interval
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, scheduler.run)
thread.join()
#all jobs must have been scheduled during 1 second
self.assertEqual(len(list(scheduler.get_jobs())), 0)
def test_get_queue_for_job_with_job_queue_name(self):
"""
Tests that scheduler gets the correct queue for the job when
queue_name is provided.
"""
queue = Queue('scheduler_foo', connection=self.testconn)
job_queue = Queue('job_foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello, queue_name='job_foo')
self.assertEqual(scheduler.get_queue_for_job(job), job_queue)
def test_get_queue_for_job_without_job_queue_name(self):
"""
Tests that scheduler gets the scheduler queue for the job
when queue name is not provided for that job.
"""
queue = Queue('scheduler_foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello)
self.assertEqual(scheduler.get_queue_for_job(job), queue)
def stop_fetching():
scheduler = Scheduler(connection=get_connection())
for j in scheduler.get_jobs():
scheduler.cancel(j)
settings = Settings() # launch scheduled jobs import datetime from brokenpromises.worker import worker from rq_scheduler import Scheduler from brokenpromises.operations import CollectNext7days, CollectNext2Months, CollectNext2Years, CollectToday, MrClean import redis conn = redis.from_url(settings.REDIS_URL) scheduler = Scheduler(connection=conn) scheduled_jobs = scheduler.get_jobs() # remove all jobs with interval for job in scheduled_jobs: if "RunAndReplaceIntTheQueuePeriodically" in job.description: scheduler.cancel(job) today = datetime.datetime.now() # net midnight next_midnight = today + datetime.timedelta(days=1) next_midnight = datetime.datetime(next_midnight.year, next_midnight.month, next_midnight.day, 0, 10) # next month year = today.year + (today.month + 1) / 12 month = today.month % 12 + 1 next_month = datetime.datetime(year, month, 1, 0, 10) #next new year next_year = datetime.datetime(today.year + 1, 1, 1, 0, 20) # enqueue periodic jobs worker.schedule_periodically(date=next_midnight, frequence="daily" , collector=CollectToday()) worker.schedule_periodically(date=next_midnight, frequence="daily" , collector=CollectNext7days())
# @desc:
from redis import Redis
from rq import Queue
from rq_scheduler import Scheduler
from datetime import datetime, timedelta
from rqtest.rq_test import redis_conn
from rqtest.worker import task2, task4, task3, task1
class A(object):
def __init__(self):
self.name = 'aniu'
scheduler = Scheduler(connection=redis_conn)
scheduler1 = Scheduler(queue_name='low', connection=redis_conn)
scheduler2 = Scheduler(queue_name='high', connection=redis_conn)
a = A()
if __name__ == "__main__":
print('a = ', a.name)
scheduler1.enqueue_in(timedelta(seconds=5), task1, a)
scheduler2.enqueue_in(timedelta(seconds=10), task3, '叁大爷好')
scheduler.enqueue_in(timedelta(seconds=15), task4, '撕大爷好')
if 0:
for job in scheduler2.get_jobs():
scheduler2.cancel(job)
print('2取消任务， 任务名={0!r}'.format(job))
class TestScheduler(RQTestCase):
def setUp(self):
super(TestScheduler, self).setUp()
self.scheduler = Scheduler(connection=self.testconn)
def test_acquire_lock(self):
"""
When scheduler acquires a lock, besides creating a key, it should
also set an expiry that's a few seconds longer than it's polling
interval so it automatically expires if scheduler is unexpectedly
terminated.
"""
key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=20)
self.assertTrue(scheduler.acquire_lock())
self.assertIn(key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(key), 30)
scheduler.remove_lock()
self.assertNotIn(key, tl(self.testconn.keys('*')))
def test_no_two_schedulers_acquire_lock(self):
"""
Ensure that no two schedulers can acquire the lock at the
same time. When removing the lock, only the scheduler which
originally acquired the lock can remove the lock.
"""
key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler1 = Scheduler(connection=self.testconn, interval=20)
scheduler2 = Scheduler(connection=self.testconn, interval=20)
self.assertTrue(scheduler1.acquire_lock())
self.assertFalse(scheduler2.acquire_lock())
self.assertIn(key, tl(self.testconn.keys('*')))
scheduler2.remove_lock()
self.assertIn(key, tl(self.testconn.keys('*')))
scheduler1.remove_lock()
self.assertNotIn(key, tl(self.testconn.keys('*')))
def test_create_job(self):
"""
Ensure that jobs are created properly.
"""
job = self.scheduler._create_job(say_hello, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job, job_from_queue)
self.assertEqual(job_from_queue.func, say_hello)
def test_create_job_with_ttl(self):
"""
Ensure that TTL is passed to RQ.
"""
job = self.scheduler._create_job(say_hello, ttl=2, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(2, job_from_queue.ttl)
def test_create_job_with_id(self):
"""
Ensure that ID is passed to RQ.
"""
job = self.scheduler._create_job(say_hello, id='id test', args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('id test', job_from_queue.id)
def test_create_job_with_description(self):
"""
Ensure that description is passed to RQ.
"""
job = self.scheduler._create_job(say_hello, description='description', args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('description', job_from_queue.description)
def test_create_job_with_timeout(self):
"""
Ensure that timeout is passed to RQ.
"""
timeout = 13
job = self.scheduler._create_job(say_hello, timeout=13, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(timeout, job_from_queue.timeout)
def test_job_not_persisted_if_commit_false(self):
"""
Ensure jobs are only saved to Redis if commit=True.
"""
job = self.scheduler._create_job(say_hello, commit=False)
self.assertEqual(self.testconn.hgetall(job.key), {})
def test_create_scheduled_job(self):
"""
Ensure that scheduled jobs are put in the scheduler queue with the right score
"""
scheduled_time = datetime.utcnow()
job = self.scheduler.enqueue_at(scheduled_time, say_hello)
self.assertEqual(job, Job.fetch(job.id, connection=self.testconn))
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(scheduled_time))
def test_create_job_with_meta(self):
"""
Ensure that meta information on the job is passed to rq
"""
expected = {'say': 'hello'}
job = self.scheduler._create_job(say_hello, meta=expected)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(expected, job_from_queue.meta)
def test_enqueue_at_sets_timeout(self):
"""
Ensure that a job scheduled via enqueue_at can be created with
a custom timeout.
"""
timeout = 13
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello, timeout=timeout)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.timeout, timeout)
def test_enqueue_at_sets_job_id(self):
"""
Ensure that a job scheduled via enqueue_at can be created with
a custom job id.
"""
job_id = 'test_id'
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello, job_id=job_id)
self.assertEqual(job.id, job_id)
def test_enqueue_at_sets_job_ttl(self):
"""
Ensure that a job scheduled via enqueue_at can be created with a custom job ttl.
"""
job_ttl = 123456789
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello, job_ttl=job_ttl)
self.assertEqual(job.ttl, job_ttl)
def test_enqueue_at_sets_job_result_ttl(self):
"""
Ensure that a job scheduled via enqueue_at can be created with a custom result ttl.
"""
job_result_ttl = 1234567890
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello, job_result_ttl=job_result_ttl)
self.assertEqual(job.result_ttl, job_result_ttl)
def test_enqueue_at_sets_meta(self):
"""
Ensure that a job scheduled via enqueue_at can be created with a custom meta.
"""
meta = {'say': 'hello'}
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello, meta=meta)
self.assertEqual(job.meta, meta)
def test_enqueue_in(self):
"""
Ensure that jobs have the right scheduled time.
"""
right_now = datetime.utcnow()
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(right_now + time_delta))
time_delta = timedelta(hours=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(right_now + time_delta))
def test_enqueue_in_sets_timeout(self):
"""
Ensure that a job scheduled via enqueue_in can be created with
a custom timeout.
"""
timeout = 13
job = self.scheduler.enqueue_in(timedelta(minutes=1), say_hello, timeout=timeout)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.timeout, timeout)
def test_enqueue_in_sets_job_id(self):
"""
Ensure that a job scheduled via enqueue_in can be created with
a custom job id.
"""
job_id = 'test_id'
job = self.scheduler.enqueue_in(timedelta(minutes=1), say_hello, job_id=job_id)
self.assertEqual(job.id, job_id)
def test_enqueue_in_sets_job_ttl(self):
"""
Ensure that a job scheduled via enqueue_in can be created with a custom job ttl.
"""
job_ttl = 123456789
job = self.scheduler.enqueue_in(timedelta(minutes=1), say_hello, job_ttl=job_ttl)
self.assertEqual(job.ttl, job_ttl)
def test_enqueue_in_sets_job_result_ttl(self):
"""
Ensure that a job scheduled via enqueue_in can be created with a custom result ttl.
"""
job_result_ttl = 1234567890
job = self.scheduler.enqueue_in(timedelta(minutes=1), say_hello, job_result_ttl=job_result_ttl)
self.assertEqual(job.result_ttl, job_result_ttl)
def test_enqueue_in_sets_meta(self):
"""
Ensure that a job scheduled via enqueue_in sets meta.
"""
meta = {'say': 'hello'}
job = self.scheduler.enqueue_in(timedelta(minutes=1), say_hello, meta=meta)
self.assertEqual(job.meta, meta)
def test_count(self):
now = datetime.utcnow()
self.scheduler.enqueue_at(now, say_hello)
self.assertEqual(self.scheduler.count(), 1)
future_time = now + timedelta(hours=1)
future_test_time = now + timedelta(minutes=59, seconds=59)
self.scheduler.enqueue_at(future_time, say_hello)
self.assertEqual(self.scheduler.count(timedelta(minutes=59, seconds=59)), 1)
self.assertEqual(self.scheduler.count(future_test_time), 1)
self.assertEqual(self.scheduler.count(), 2)
def test_get_jobs(self):
"""
Ensure get_jobs() returns all jobs until the specified time.
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs(now))
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertIn(job, self.scheduler.get_jobs(timedelta(hours=1, seconds=1)))
self.assertIn(job, [j[0] for j in self.scheduler.get_jobs(with_times=True)])
self.assertIsInstance(list(self.scheduler.get_jobs(with_times=True))[0][1], datetime)
self.assertNotIn(job, self.scheduler.get_jobs(timedelta(minutes=59, seconds=59)))
def test_get_jobs_slice(self):
"""
Ensure get_jobs() returns the appropriate slice of all jobs using offset and length.
"""
now = datetime.utcnow()
future_time = now + timedelta(hours=1)
future_test_time = now + timedelta(minutes=59, seconds=59)
# Schedule each job a second later than the previous job,
# otherwise Redis will return jobs that have the same scheduled time in
# lexicographical order (not the order in which we enqueued them)
now_jobs = [self.scheduler.enqueue_at(now + timedelta(seconds=x), say_hello)
for x in range(15)]
future_jobs = [self.scheduler.enqueue_at(future_time + timedelta(seconds=x), say_hello)
for x in range(15)]
expected_slice = now_jobs[5:] + future_jobs[:10] # last 10 from now_jobs and first 10 from future_jobs
expected_until_slice = now_jobs[5:] # last 10 from now_jobs
jobs = self.scheduler.get_jobs()
jobs_slice = self.scheduler.get_jobs(offset=5, length=20)
jobs_until_slice = self.scheduler.get_jobs(future_test_time, offset=5, length=20)
self.assertEqual(now_jobs + future_jobs, list(jobs))
self.assertEqual(expected_slice, list(jobs_slice))
self.assertEqual(expected_until_slice, list(jobs_until_slice))
def test_get_jobs_to_queue(self):
"""
Ensure that jobs scheduled the future are not queued.
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertNotIn(job, self.scheduler.get_jobs_to_queue())
def test_enqueue_job(self):
"""
When scheduled job is enqueued, make sure:
- Job is removed from the sorted set of scheduled jobs
- "enqueued_at" attribute is properly set
- Job appears in the right queue
- Queue is recognized by rq's Queue.all()
"""
now = datetime.utcnow()
queue_name = 'foo'
scheduler = Scheduler(connection=self.testconn, queue_name=queue_name)
job = scheduler.enqueue_at(now, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(job, tl(self.testconn.zrange(scheduler.scheduled_jobs_key, 0, 10)))
job = Job.fetch(job.id, connection=self.testconn)
self.assertTrue(job.enqueued_at is not None)
queue = scheduler.get_queue_for_job(job)
self.assertIn(job, queue.jobs)
queue = Queue.from_queue_key('rq:queue:{0}'.format(queue_name))
self.assertIn(job, queue.jobs)
self.assertIn(queue, Queue.all())
def test_enqueue_job_with_queue(self):
"""
Ensure that job is enqueued correctly when the scheduler is bound
to a queue object.
"""
queue = Queue('foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello)
scheduler_queue = scheduler.get_queue_for_job(job)
self.assertEqual(queue, scheduler_queue)
scheduler.enqueue_job(job)
self.assertTrue(job.enqueued_at is not None)
self.assertIn(job, queue.jobs)
self.assertIn(queue, Queue.all())
def test_job_membership(self):
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler)
self.assertIn(job.id, self.scheduler)
self.assertNotIn("non-existing-job-id", self.scheduler)
def test_cancel_scheduled_job(self):
"""
When scheduled job is canceled, make sure:
- Job is removed from the sorted set of scheduled jobs
"""
# schedule a job to be enqueued one minute from now
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
# cancel the scheduled job and check that it's gone from the set
self.scheduler.cancel(job)
self.assertNotIn(job.id, tl(self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1)))
def test_change_execution_time(self):
"""
Ensure ``change_execution_time`` is called, ensure that job's score is updated
"""
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello)
new_date = datetime(2010, 1, 1)
self.scheduler.change_execution_time(job, new_date)
self.assertEqual(to_unix(new_date),
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
self.scheduler.cancel(job)
self.assertRaises(ValueError, self.scheduler.change_execution_time, job, new_date)
def test_args_kwargs_are_passed_correctly(self):
"""
Ensure that arguments and keyword arguments are properly saved to jobs.
"""
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, z=1, y=1, x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, 1, z=1, y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1,))
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_in(time_delta, simple_addition, z=1, y=1, x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, z=1, y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1,))
def test_interval_and_repeat_persisted_correctly(self):
"""
Ensure that interval and repeat attributes are correctly saved.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=10, repeat=11)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.meta['interval'], 10)
self.assertEqual(job_from_queue.meta['repeat'], 11)
def test_crontab_persisted_correctly(self):
"""
Ensure that crontab attribute gets correctly saved in Redis.
"""
# create a job that runs one minute past each whole hour
job = self.scheduler.cron("1 * * * *", say_hello)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.meta['cron_string'], "1 * * * *")
# get the scheduled_time and convert it to a datetime object
unix_time = self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id)
datetime_time = from_unix(unix_time)
# check that minute=1, seconds=0, and is within an hour
assert datetime_time.minute == 1
assert datetime_time.second == 0
assert datetime_time - datetime.utcnow() < timedelta(hours=1)
def test_crontab_sets_timeout(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom timeout.
"""
timeout = 13
job = self.scheduler.cron("1 * * * *", say_hello, timeout=timeout)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.timeout, timeout)
def test_crontab_sets_id(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom id
"""
job_id = "hello-job-id"
job = self.scheduler.cron("1 * * * *", say_hello, id=job_id)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_id, job_from_queue.id)
def test_crontab_sets_default_result_ttl(self):
"""
Ensure that a job scheduled via crontab gets proper default
result_ttl (-1) periodic tasks.
"""
job = self.scheduler.cron("1 * * * *", say_hello)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(-1, job_from_queue.result_ttl)
def test_crontab_sets_description(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom description
"""
description = 'test description'
job = self.scheduler.cron("1 * * * *", say_hello, description=description)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(description, job_from_queue.description)
def test_repeat_without_interval_raises_error(self):
# Ensure that an error is raised if repeat is specified without interval
def create_job():
self.scheduler.schedule(datetime.utcnow(), say_hello, repeat=11)
self.assertRaises(ValueError, create_job)
def test_job_with_intervals_get_rescheduled(self):
"""
Ensure jobs with interval attribute are put back in the scheduler
"""
time_now = datetime.utcnow()
interval = 10
job = self.scheduler.schedule(time_now, say_hello, interval=interval)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(time_now) + interval)
def test_job_with_interval_can_set_meta(self):
"""
Ensure that jobs with interval attribute can be created with meta
"""
time_now = datetime.utcnow()
interval = 10
meta = {'say': 'hello'}
job = self.scheduler.schedule(time_now, say_hello, interval=interval, meta=meta)
self.scheduler.enqueue_job(job)
self.assertEqual(job.meta, meta)
def test_job_with_crontab_get_rescheduled(self):
# Create a job with a cronjob_string
job = self.scheduler.cron("1 * * * *", say_hello)
# current unix_time
old_next_scheduled_time = self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id)
# change crontab
job.meta['cron_string'] = "2 * * * *"
# enqueue the job
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# check that next scheduled time has changed
self.assertNotEqual(old_next_scheduled_time,
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
# check that new next scheduled time is set correctly
expected_next_scheduled_time = to_unix(get_next_scheduled_time("2 * * * *"))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
expected_next_scheduled_time)
def test_job_with_repeat(self):
"""
Ensure jobs with repeat attribute are put back in the scheduler
X (repeat) number of times
"""
time_now = datetime.utcnow()
interval = 10
# If job is repeated once, the job shouldn't be put back in the queue
job = self.scheduler.schedule(time_now, say_hello, interval=interval, repeat=1)
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.schedule(time_now, say_hello, interval=interval, repeat=2)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
def test_missing_jobs_removed_from_scheduler(self):
"""
Ensure jobs that don't exist when queued are removed from the scheduler.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello)
job.cancel()
list(self.scheduler.get_jobs_to_queue())
self.assertIn(job.id, tl(self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1)))
job.delete()
list(self.scheduler.get_jobs_to_queue())
self.assertNotIn(job.id, tl(self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1)))
def test_periodic_jobs_sets_result_ttl(self):
"""
Ensure periodic jobs set result_ttl to infinite.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.result_ttl, -1)
def test_periodic_jobs_sets_ttl(self):
"""
Ensure periodic jobs sets correctly ttl.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5, ttl=4)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.ttl, 4)
def test_periodic_jobs_sets_meta(self):
"""
Ensure periodic jobs sets correctly meta.
"""
meta = {'say': 'hello'}
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5, meta=meta)
self.assertEqual(meta, job.meta)
def test_periodic_job_sets_id(self):
"""
Ensure that ID is passed to RQ by schedule.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5, id='id test')
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('id test', job.id)
def test_periodic_job_sets_description(self):
"""
Ensure that description is passed to RQ by schedule.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5, description='description')
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('description', job.description)
def test_run(self):
"""
Check correct signal handling in Scheduler.run().
"""
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, self.scheduler.run)
thread.join()
def test_run_burst(self):
"""
Check burst mode of Scheduler.run().
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
self.assertEqual(len(list(self.scheduler.get_jobs())), 1)
self.scheduler.run(burst=True)
self.assertEqual(len(list(self.scheduler.get_jobs())), 0)
def test_scheduler_w_o_explicit_connection(self):
"""
Ensure instantiating Scheduler w/o explicit connection works.
"""
s = Scheduler()
self.assertEqual(s.connection, self.testconn)
def test_small_float_interval(self):
"""
Test that scheduler accepts 'interval' of type float, less than 1 second.
"""
key = Scheduler.scheduler_key
lock_key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=0.1) # testing interval = 0.1 second
self.assertEqual(scheduler._interval, 0.1)
#acquire lock
self.assertTrue(scheduler.acquire_lock())
self.assertIn(lock_key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(lock_key), 10) # int(0.1) + 10 = 10
#enqueue a job
now = datetime.utcnow()
job = scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
self.assertEqual(len(list(self.scheduler.get_jobs())), 1)
#remove the lock
scheduler.remove_lock()
#test that run works with the small floating-point interval
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, scheduler.run)
thread.join()
#all jobs must have been scheduled during 1 second
self.assertEqual(len(list(scheduler.get_jobs())), 0)
def cancel(self):
logger.info('cancelling future Notification pk {}'.format(self.pk))
scheduler = Scheduler(connection=Redis())
scheduler.cancel(self.job_id)
self.canceled = True
self.save()
def cancel_job_view(job_id):
scheduler = Scheduler()
scheduler.cancel(job_id)
return dict(status='OK')
import os
import redis
# from rq import Queue
from rq_scheduler import Scheduler
from datetime import datetime
from app import loop_script
redis_url = os.getenv('REDISCLOUD_URL', 'redis://localhost:6379')
conn = redis.from_url(redis_url)
# q = Queue(connection=conn)
# result = q.enqueue(loop_script)
scheduler = Scheduler(connection=conn)
if len(scheduler.get_jobs()) > 0:
for job in scheduler.get_jobs():
scheduler.cancel(job)
scheduler.schedule(scheduled_time=datetime.utcnow(),
func=loop_script,
interval=30)
print(scheduler.get_jobs())
def schedules(org_label, project_label, schedule_label):
''' schedule configuration management
/organizations/aquaya/projects/water-quality/schedules
: viewing a list of this project's schedules
/organizations/aquaya/projects/water-quality/schedules?create=true
: create a new schedule config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/schedules/weekly
: view the 'weekly' schedule config
/organizations/aquaya/projects/water-quality/schedules/weekly?edit=true
: edit a schedule config; accepts GET or POST
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'POST':
# we have a schedule_label
schedules = Schedule.objects(label=schedule_label, project=project)
if not schedules:
abort(404)
schedule = schedules[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if schedule.name != request.form.get('name', ''):
name = request.form.get('name', '')
schedule.name = name
schedules = Schedule.objects(project=project).only('label')
labels = [s.label for s in schedules]
schedule.label = utilities.generate_label(name, labels)
schedule.description = request.form.get('description', '')
elif form_type == 'items':
# modify schedule attachments
# 'items' is of the form report_id__4abcd00123 or statistic_id..
# convert these to real objects
items = request.form.getlist('items')
attached_reports, attached_statistics = [], []
for item in items:
prefix, item_id = item.split('__')
item_type = prefix.split('_')[0]
if item_type == 'report':
reports = Report.objects(id=item_id)
if not reports:
abort(404)
attached_reports.append(reports[0])
elif item_type == 'statistic':
statistics = Statistic.objects(id=item_id)
if not statistics:
abort(404)
attached_statistics.append(statistics[0])
schedule.update(set__reports=attached_reports)
schedule.update(set__statistics=attached_statistics)
# whether we're also sending project data
send_data = request.form.get('send_project_data', '')
if send_data == 'true':
schedule.update(set__send_project_data = True)
else:
schedule.update(set__send_project_data = False)
# save the list of project-data-filters
# filters are formatted like 'filter_id__4abcd123'
filter_ids = request.form.getlist('filters')
attached_filters = []
for filter_id in filter_ids:
filters = Filter.objects(id=filter_id.split('__')[1])
if not filters:
abort(404)
attached_filters.append(filters[0])
schedule.update(set__data_filters = attached_filters)
# how to apply the filters
apply_any_filters = request.form.get('apply_any_filters', '')
if apply_any_filters == 'true':
schedule.update(set__apply_any_filters= True)
else:
schedule.update(set__apply_any_filters= False)
flash('Schedules attachments saved.', 'success')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#items')
elif form_type == 'schedule':
old_interval = schedule.interval
new_interval = {
'type': request.form.get('schedule_type', '')
, 'at': request.form.get('send_at', '')
, 'on_day': request.form.get('send_on_day', '')}
if old_interval != new_interval:
schedule.interval = new_interval
schedule.save()
schedule.reload()
# delete any old jobs and schedule the new ones
update_scheduled_send(schedule.id)
flash('Scheduling interval saved successfully.', 'success')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#schedule')
elif form_type == 'message':
message_type = request.form.get('message_type', '')
schedule.message_type = message_type
if message_type == 'email':
# attach email info to schedule
schedule.email_subject = request.form.get('email_subject', '')
schedule.email_body = request.form.get('email_body', '')
add_email = request.form.get('add_email_recipient_email', '')
if add_email:
# is the email already in place? (shouldn't be)
for recipient in schedule.email_recipients:
if recipient['email'] == add_email:
flash('"%s" is already included in this \
schedule' % add_email, 'warning')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#message')
else:
# add the recipient to the schedule
add_name = request.form.get(
'add_email_recipient_name', '')
recipient = {'email': add_email, 'name': add_name}
schedule.update(push__email_recipients=recipient)
flash('Successfully added "%s" to this schedule' \
% add_email, 'success')
remove_email = request.form.get('remove_email_recipient', '')
if remove_email:
# is the email already attached? (should be)
emails = [r['email'] for r in schedule.email_recipients]
if remove_email not in emails:
flash('"%s" is not included in this schedule and \
cannot be removed' % remove_email, 'warning')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#message')
else:
# remove it
for recipient in schedule.email_recipients:
if recipient['email'] == remove_email:
schedule.update(
pull__email_recipients=recipient)
flash('Successfully removed the address "%s" \
from this schedule' % remove_email
, 'success')
elif message_type == 'sms':
# attach sms info to schedule
add_number = request.form.get(
'add_sms_recipient_phone_number', '')
if add_number:
# is the number already in place? (shouldn't be)
for recipient in schedule.sms_recipients:
if recipient['phone_number'] == add_number:
flash('"%s" is already included in this \
schedule' % add_number, 'warning')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#message')
else:
# add the recipient to the schedule
add_name = request.form.get(
'add_sms_recipient_name', '')
recipient = {'phone_number': add_number
, 'name': add_name}
schedule.update(push__sms_recipients=recipient)
flash('Successfully added "%s" to this schedule' \
% add_number, 'success')
remove_number = request.form.get('remove_sms_recipient', '')
if remove_number:
# is the number already attached? (should be)
numbers = [r['phone_number'] for r in \
schedule.sms_recipients]
if remove_number not in numbers:
flash('"%s" is not included in this SMS schedule and \
cannot be removed' % remove_number, 'warning')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#message')
else:
# remove it
for recipient in schedule.sms_recipients:
if recipient['phone_number'] == remove_number:
schedule.update(
pull__sms_recipients=recipient)
flash('Successfully removed the address "%s" \
from this schedule' % remove_number
, 'success')
break
schedule.save()
flash('Message settings changed successfully.', 'success')
url = url_for('schedules', org_label=org.label
, project_label=project.label, schedule_label=schedule.label
, edit='true')
return redirect(url + '#message')
elif form_type == 'admin':
# delete the schedule
# keeping the associated messages as they're an important log
# related items in the queue will not fire sans schedule
name = schedule.name
# cancel any outstanding jobs
if schedule.next_task_id:
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host'], redis_config['port']
, password=redis_config['password']))
scheduler = Scheduler()
job = Job(id=schedule.next_task_id)
scheduler.cancel(job)
# delete the job itself
schedule.delete()
app.logger.info('%s deleted schdedule "%s"' % \
(session['email'], name))
flash('schedule "%s" was deleted successfully' % name, 'success')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
schedule.save()
flash('Changes saved successfully.', 'success')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true'))
except:
app.logger.error('%s experienced an error saving info about %s' % (
session['email'], request.form['name']))
flash('Error saving changes - make sure schedule names are \
unique.', 'error')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label, schedule_label=schedule_label
, edit='true'))
if request.method == 'GET':
if schedule_label:
schedules = Schedule.objects(label=schedule_label, project=project)
if not schedules:
app.logger.error('%s tried to access a schedule that does \
not exist' % session['email'])
flash('schedule "%s" not found, sorry!' % schedule_label
, 'warning')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label))
schedule = schedules[0]
if request.args.get('edit', '') == 'true':
available_reports = Report.objects(project=project)
available_filters = Filter.objects(project=project)
available_statistics = Statistic.objects(project=project)
emails = json.dumps(schedule.email_recipients)
current_time = datetime.datetime.utcnow()
return render_template('schedule_edit.html', schedule=schedule
, available_reports=available_reports
, available_filters=available_filters
, available_statistics=available_statistics
, email_recipients=emails
, current_time = current_time)
elif request.args.get('fire', '') == 'true':
if ready_to_send(schedule):
# immediately enqueue the job with rq
# countdown briefly to ensure next_task_id can be updated?
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host']
, redis_config['port']
, password=redis_config['password']))
queue = Queue()
job = queue.enqueue(send_scheduled_report, schedule.id)
# save the id
# race condition with enqueued func above?
# that's why we used to have a countdown
schedule.update(set__next_task_id = job.id)
flash('Sending message, this may take a few \
moments. Check the message log for updates.', 'info')
else:
flash('This schedule is not properly configured.', 'error')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule_label))
else:
# get all messages attached to this schedule
messages = Message.objects(schedule=schedule).order_by(
'-creation_time')
return render_template('schedule.html', schedule=schedule
, messages=messages)
if request.args.get('create', '') == 'true':
# create a new schedule
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
schedule_name = 'schd-%s' \
% utilities.generate_random_string(6)
new_schedule = Schedule(
label = schedule_name.lower()
, project = project
, name = schedule_name
)
new_schedule.save()
app.logger.info('schedule created by %s' % session['email'])
flash('schedule created. Please change the default values.'
, 'success')
except:
app.logger.error('schedule creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project=project.label))
# redirect to the editing screen
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=new_schedule.label, edit='true'))
# no statistic in particular was specified; show em all
schedules = Schedule.objects(project=project)
return render_template('project_schedules.html', project=project
, schedules=schedules)
class TestScheduler(RQTestCase):
def setUp(self):
super(TestScheduler, self).setUp()
self.scheduler = Scheduler(connection=self.testconn)
def test_birth_and_death_registration(self):
"""
When scheduler registers it's birth, besides creating a key, it should
also set an expiry that's a few seconds longer than it's polling
interval so it automatically expires if scheduler is unexpectedly
terminated.
"""
key = Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=20)
scheduler.register_birth()
self.assertIn(key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(key), 30)
self.assertFalse(self.testconn.hexists(key, 'death'))
self.assertRaises(ValueError, scheduler.register_birth)
scheduler.register_death()
self.assertTrue(self.testconn.hexists(key, 'death'))
def test_create_job(self):
"""
Ensure that jobs are created properly.
"""
job = self.scheduler._create_job(say_hello, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job, job_from_queue)
self.assertEqual(job_from_queue.func, say_hello)
def test_create_job_with_ttl(self):
"""
Ensure that TTL is passed to RQ.
"""
job = self.scheduler._create_job(say_hello, ttl=2, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(2, job_from_queue.ttl)
def test_create_job_with_id(self):
"""
Ensure that ID is passed to RQ.
"""
job = self.scheduler._create_job(say_hello, id='id test', args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('id test', job_from_queue.id)
def test_create_job_with_description(self):
"""
Ensure that description is passed to RQ.
"""
job = self.scheduler._create_job(say_hello, description='description', args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('description', job_from_queue.description)
def test_job_not_persisted_if_commit_false(self):
"""
Ensure jobs are only saved to Redis if commit=True.
"""
job = self.scheduler._create_job(say_hello, commit=False)
self.assertEqual(self.testconn.hgetall(job.key), {})
def test_create_scheduled_job(self):
"""
Ensure that scheduled jobs are put in the scheduler queue with the right score
"""
scheduled_time = datetime.utcnow()
job = self.scheduler.enqueue_at(scheduled_time, say_hello)
self.assertEqual(job, Job.fetch(job.id, connection=self.testconn))
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(scheduled_time))
def test_enqueue_in(self):
"""
Ensure that jobs have the right scheduled time.
"""
right_now = datetime.utcnow()
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(right_now + time_delta))
time_delta = timedelta(hours=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(right_now + time_delta))
def test_get_jobs(self):
"""
Ensure get_jobs() returns all jobs until the specified time.
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs(now))
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertIn(job, self.scheduler.get_jobs(timedelta(hours=1, seconds=1)))
self.assertIn(job, [j[0] for j in self.scheduler.get_jobs(with_times=True)])
self.assertIsInstance(self.scheduler.get_jobs(with_times=True)[0][1], datetime)
self.assertNotIn(job, self.scheduler.get_jobs(timedelta(minutes=59, seconds=59)))
def test_get_jobs_to_queue(self):
"""
Ensure that jobs scheduled the future are not queued.
"""
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertNotIn(job, self.scheduler.get_jobs_to_queue())
def test_enqueue_job(self):
"""
When scheduled job is enqueued, make sure:
- Job is removed from the sorted set of scheduled jobs
- "enqueued_at" attribute is properly set
- Job appears in the right queue
- Queue is recognized by rq's Queue.all()
"""
now = datetime.utcnow()
queue_name = 'foo'
scheduler = Scheduler(connection=self.testconn, queue_name=queue_name)
job = scheduler.enqueue_at(now, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(job, tl(self.testconn.zrange(scheduler.scheduled_jobs_key, 0, 10)))
job = Job.fetch(job.id, connection=self.testconn)
self.assertTrue(job.enqueued_at is not None)
queue = scheduler.get_queue_for_job(job)
self.assertIn(job, queue.jobs)
queue = Queue.from_queue_key('rq:queue:{0}'.format(queue_name))
self.assertIn(job, queue.jobs)
self.assertIn(queue, Queue.all())
def test_job_membership(self):
now = datetime.utcnow()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler)
self.assertIn(job.id, self.scheduler)
self.assertNotIn("non-existing-job-id", self.scheduler)
def test_cancel_scheduled_job(self):
"""
When scheduled job is canceled, make sure:
- Job is removed from the sorted set of scheduled jobs
"""
# schedule a job to be enqueued one minute from now
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
# cancel the scheduled job and check that it's gone from the set
self.scheduler.cancel(job)
self.assertNotIn(job.id, tl(self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1)))
def test_change_execution_time(self):
"""
Ensure ``change_execution_time`` is called, ensure that job's score is updated
"""
job = self.scheduler.enqueue_at(datetime.utcnow(), say_hello)
new_date = datetime(2010, 1, 1)
self.scheduler.change_execution_time(job, new_date)
self.assertEqual(to_unix(new_date),
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
self.scheduler.cancel(job)
self.assertRaises(ValueError, self.scheduler.change_execution_time, job, new_date)
def test_args_kwargs_are_passed_correctly(self):
"""
Ensure that arguments and keyword arguments are properly saved to jobs.
"""
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, z=1, y=1, x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_at(datetime.utcnow(), simple_addition, 1, z=1, y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1,))
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_in(time_delta, simple_addition, z=1, y=1, x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, z=1, y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1,))
def test_enqueue_is_deprecated(self):
"""
Ensure .enqueue() throws a DeprecationWarning
"""
with warnings.catch_warnings(record=True) as w:
# Enable all warnings
warnings.simplefilter("always")
job = self.scheduler.enqueue(datetime.utcnow(), say_hello)
self.assertEqual(1, len(w))
self.assertEqual(w[0].category, DeprecationWarning)
def test_enqueue_periodic(self):
"""
Ensure .enqueue_periodic() throws a DeprecationWarning
"""
with warnings.catch_warnings(record=True) as w:
# Enable all warnings
warnings.simplefilter("always")
job = self.scheduler.enqueue_periodic(datetime.utcnow(), 1, None, say_hello)
self.assertEqual(1, len(w))
self.assertEqual(w[0].category, DeprecationWarning)
def test_interval_and_repeat_persisted_correctly(self):
"""
Ensure that interval and repeat attributes get correctly saved in Redis.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=10, repeat=11)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.meta['interval'], 10)
self.assertEqual(job_from_queue.meta['repeat'], 11)
def test_crontab_persisted_correctly(self):
"""
Ensure that crontab attribute gets correctly saved in Redis.
"""
# create a job that runs one minute past each whole hour
job = self.scheduler.cron("1 * * * *", say_hello)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.meta['cron_string'], "1 * * * *")
# get the scheduled_time and convert it to a datetime object
unix_time = self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id)
datetime_time = from_unix(unix_time)
# check that minute=1, seconds=0, and is within an hour
assert datetime_time.minute == 1
assert datetime_time.second == 0
assert datetime_time - datetime.utcnow() < timedelta(hours=1)
def test_crontab_sets_timeout(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom timeout.
"""
timeout = 13
job = self.scheduler.cron("1 * * * *", say_hello, timeout=timeout)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_from_queue.timeout, timeout)
def test_crontab_sets_id(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom id
"""
job_id = "hello-job-id"
job = self.scheduler.cron("1 * * * *", say_hello, id=job_id)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job_id, job_from_queue.id)
def test_crontab_sets_default_result_ttl(self):
"""
Ensure that a job scheduled via crontab gets proper default
result_ttl (-1) periodic tasks.
"""
job = self.scheduler.cron("1 * * * *", say_hello)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(-1, job_from_queue.result_ttl)
def test_crontab_sets_description(self):
"""
Ensure that a job scheduled via crontab can be created with
a custom description
"""
description = 'test description'
job = self.scheduler.cron("1 * * * *", say_hello, description=description)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(description, job_from_queue.description)
def test_repeat_without_interval_raises_error(self):
# Ensure that an error is raised if repeat is specified without interval
def create_job():
self.scheduler.schedule(datetime.utcnow(), say_hello, repeat=11)
self.assertRaises(ValueError, create_job)
def test_job_with_intervals_get_rescheduled(self):
"""
Ensure jobs with interval attribute are put back in the scheduler
"""
time_now = datetime.utcnow()
interval = 10
job = self.scheduler.schedule(time_now, say_hello, interval=interval)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(time_now) + interval)
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, None, say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
to_unix(time_now) + interval)
def test_job_with_crontab_get_rescheduled(self):
# Create a job with a cronjob_string
job = self.scheduler.cron("1 * * * *", say_hello)
# current unix_time
old_next_scheduled_time = self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id)
# change crontab
job.meta['cron_string'] = "2 * * * *"
# enqueue the job
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# check that next scheduled time has changed
self.assertNotEqual(old_next_scheduled_time,
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
# check that new next scheduled time is set correctly
expected_next_scheduled_time = to_unix(get_next_scheduled_time("2 * * * *"))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
expected_next_scheduled_time)
def test_job_with_repeat(self):
"""
Ensure jobs with repeat attribute are put back in the scheduler
X (repeat) number of times
"""
time_now = datetime.utcnow()
interval = 10
# If job is repeated once, the job shouldn't be put back in the queue
job = self.scheduler.schedule(time_now, say_hello, interval=interval, repeat=1)
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.schedule(time_now, say_hello, interval=interval, repeat=2)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
time_now = datetime.utcnow()
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, 1, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.enqueue_periodic(time_now, interval, 2, say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
tl(self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)))
def test_missing_jobs_removed_from_scheduler(self):
"""
Ensure jobs that don't exist when queued are removed from the scheduler.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello)
job.cancel()
self.scheduler.get_jobs_to_queue()
self.assertNotIn(job.id, tl(self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1)))
def test_periodic_jobs_sets_result_ttl(self):
"""
Ensure periodic jobs set result_ttl to infinite.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.result_ttl, -1)
def test_periodic_jobs_sets_ttl(self):
"""
Ensure periodic jobs sets correctly ttl.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5, ttl=4)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.ttl, 4)
def test_periodic_job_sets_id(self):
"""
Ensure that ID is passed to RQ by schedule.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5, id='id test')
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('id test', job.id)
def test_periodic_job_sets_description(self):
"""
Ensure that description is passed to RQ by schedule.
"""
job = self.scheduler.schedule(datetime.utcnow(), say_hello, interval=5, description='description')
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual('description', job.description)
def test_run(self):
"""
Check correct signal handling in Scheduler.run().
"""
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, self.scheduler.run)
thread.join()
def test_scheduler_w_o_explicit_connection(self):
"""
Ensure instantiating Scheduler w/o explicit connection works.
"""
s = Scheduler()
self.assertEqual(s.connection, self.testconn)
def test_small_float_interval(self):
"""
Test that scheduler accepts 'interval' of type float, less than 1 second.
"""
key = Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=0.1) # testing interval = 0.1 second
self.assertEqual(scheduler._interval, 0.1)
#register birth
scheduler.register_birth()
self.assertIn(key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(key), 10) # int(0.1) + 10 = 10
self.assertFalse(self.testconn.hexists(key, 'death'))
#enqueue a job
now = datetime.utcnow()
job = scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
self.assertEqual(len(self.scheduler.get_jobs()), 1)
#register death
scheduler.register_death()
#test that run works with the small floating-point interval
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, scheduler.run)
thread.join()
#all jobs must have been scheduled during 1 second
self.assertEqual(len(scheduler.get_jobs()), 0)
class TestScheduler(RQTestCase):
def setUp(self):
super(TestScheduler, self).setUp()
self.scheduler = Scheduler(connection=self.testconn)
def test_birth_and_death_registration(self):
key = Scheduler.scheduler_key
self.assertNotIn(key, self.testconn.keys('*'))
scheduler = Scheduler(connection=self.testconn)
scheduler.register_birth()
self.assertIn(key, self.testconn.keys('*'))
self.assertFalse(self.testconn.hexists(key, 'death'))
self.assertRaises(ValueError, scheduler.register_birth)
scheduler.register_death()
self.assertTrue(self.testconn.hexists(key, 'death'))
def test_create_job(self):
"""
Ensure that jobs are created properly.
"""
job = self.scheduler._create_job(say_hello, args=(), kwargs={})
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job, job_from_queue)
self.assertEqual(job_from_queue.func, say_hello)
def test_job_not_persisted_if_commit_false(self):
"""
Ensure jobs are only saved to Redis if commit=True.
"""
job = self.scheduler._create_job(say_hello, commit=False)
self.assertEqual(self.testconn.hgetall(job.key), {})
def test_create_scheduled_job(self):
"""
Ensure that scheduled jobs are put in the scheduler queue with the right score
"""
scheduled_time = datetime.now()
job = self.scheduler.enqueue_at(scheduled_time, say_hello)
self.assertEqual(job, Job.fetch(job.id, connection=self.testconn))
self.assertIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int(scheduled_time.strftime('%s')))
def test_enqueue_in(self):
"""
Ensure that jobs have the right scheduled time.
"""
right_now = datetime.now()
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertIn(job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int((right_now + time_delta).strftime('%s')))
time_delta = timedelta(hours=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int((right_now + time_delta).strftime('%s')))
def test_get_jobs_to_queue(self):
"""
Ensure that jobs scheduled the future are not queued.
"""
now = datetime.now()
job = self.scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
future_time = now + timedelta(hours=1)
job = self.scheduler.enqueue_at(future_time, say_hello)
self.assertNotIn(job, self.scheduler.get_jobs_to_queue())
def test_enqueue_job(self):
"""
When scheduled job is enqueued, make sure:
- Job is removed from the sorted set of scheduled jobs
- "enqueued_at" attribute is properly set
- Job appears in the right queue
"""
now = datetime.now()
queue_name = 'foo'
scheduler = Scheduler(connection=self.testconn, queue_name=queue_name)
job = scheduler.enqueue_at(now, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(job, self.testconn.zrange(scheduler.scheduled_jobs_key, 0, 10))
job = Job.fetch(job.id, connection=self.testconn)
self.assertTrue(job.enqueued_at is not None)
queue = scheduler.get_queue_for_job(job)
self.assertIn(job, queue.jobs)
queue = Queue.from_queue_key('rq:queue:{0}'.format(queue_name))
self.assertIn(job, queue.jobs)
def test_cancel_scheduled_job(self):
"""
When scheduled job is canceled, make sure:
- Job is removed from the sorted set of scheduled jobs
"""
# schedule a job to be enqueued one minute from now
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
# cancel the scheduled job and check that it's gone from the set
self.scheduler.cancel(job)
self.assertNotIn(job.id, self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1))
def test_change_execution_time(self):
"""
Ensure ``change_execution_time`` is called, ensure that job's score is updated
"""
job = self.scheduler.enqueue_at(datetime.now(), say_hello)
new_date = datetime(2010, 1, 1)
self.scheduler.change_execution_time(job, new_date)
self.assertEqual(int(new_date.strftime('%s')),
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
def test_args_kwargs_are_passed_correctly(self):
"""
Ensure that arguments and keyword arguments are properly saved to jobs.
"""
job = self.scheduler.enqueue_at(datetime.now(), simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_at(datetime.now(), simple_addition, z=1, y=1, x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_at(datetime.now(), simple_addition, 1, z=1, y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1,))
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, 1, 1)
self.assertEqual(job.args, (1, 1, 1))
job = self.scheduler.enqueue_in(time_delta, simple_addition, z=1, y=1, x=1)
self.assertEqual(job.kwargs, {'x': 1, 'y': 1, 'z': 1})
job = self.scheduler.enqueue_in(time_delta, simple_addition, 1, z=1, y=1)
self.assertEqual(job.kwargs, {'y': 1, 'z': 1})
self.assertEqual(job.args, (1,))
def test_interval_and_repeat_persisted_correctly(self):
"""
Ensure that interval and repeat attributes get correctly saved in Redis.
"""
job = self.scheduler.enqueue(datetime.now(), say_hello, interval=10, repeat=11)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(int(job_from_queue.interval), 10)
self.assertEqual(int(job_from_queue.repeat), 11)
def test_repeat_without_interval_raises_error(self):
# Ensure that an error is raised if repeat is specified without interval
def create_job():
self.scheduler.enqueue(datetime.now(), say_hello, repeat=11)
self.assertRaises(ValueError, create_job)
def test_job_with_intervals_get_rescheduled(self):
"""
Ensure jobs with interval attribute are put back in the scheduler
"""
time_now = datetime.now()
interval = 10
job = self.scheduler.enqueue(time_now, say_hello, interval=interval)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int(time_now.strftime('%s')) + interval)
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, None, say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
int(time_now.strftime('%s')) + interval)
def test_job_with_repeat(self):
"""
Ensure jobs with repeat attribute are put back in the scheduler
X (repeat) number of times
"""
time_now = datetime.now()
interval = 10
# If job is repeated once, the job shouldn't be put back in the queue
job = self.scheduler.enqueue(time_now, say_hello, interval=interval, repeat=1)
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.enqueue(time_now, say_hello, interval=interval, repeat=2)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
time_now = datetime.now()
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, 1, say_hello)
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
# If job is repeated twice, it should only be put back in the queue once
job = self.scheduler.enqueue_periodic(time_now, interval, 2, say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.scheduler.enqueue_job(job)
self.assertNotIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
def test_missing_jobs_removed_from_scheduler(self):
"""
Ensure jobs that don't exist when queued are removed from the scheduler.
"""
job = self.scheduler.enqueue(datetime.now(), say_hello)
job.cancel()
self.scheduler.get_jobs_to_queue()
self.assertNotIn(job.id, self.testconn.zrange(
self.scheduler.scheduled_jobs_key, 0, 1))
def test_periodic_jobs_sets_ttl(self):
"""
Ensure periodic jobs set result_ttl to infinite.
"""
job = self.scheduler.enqueue(datetime.now(), say_hello, interval=5)
job_from_queue = Job.fetch(job.id, connection=self.testconn)
self.assertEqual(job.result_ttl, -1)
