def test_job_with_intervals_get_rescheduled(self):
"""
Ensure jobs with interval attribute are put back in the scheduler
"""
time_now = times.now()
interval = 10
job = self.scheduler.schedule(time_now, say_hello, interval=interval)
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(time_now) + interval)
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, None,
say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(time_now) + interval)
def get_jobs(self, until=None, with_times=False):
"""
Returns a list of job instances that will be queued until the given time.
If no 'until' argument is given all jobs are returned. This function
accepts datetime and timedelta instances as well as integers representing
epoch values.
If with_times is True a list of tuples consisting of the job instance and
it's scheduled execution time is returned.
"""
def epoch_to_datetime(epoch):
return datetime.fromtimestamp(float(epoch))
if until is None:
until = "+inf"
elif isinstance(until, datetime):
until = times.to_unix(until)
elif isinstance(until, timedelta):
until = times.to_unix((times.now() + until))
job_ids = self.connection.zrangebyscore(self.scheduled_jobs_key, 0,
until, withscores=with_times,
score_cast_func=epoch_to_datetime)
if not with_times:
job_ids = zip(job_ids, repeat(None))
jobs = []
for job_id, sched_time in job_ids:
try:
job = Job.fetch(job_id, connection=self.connection)
if with_times:
jobs.append((job, sched_time))
else:
jobs.append(job)
except NoSuchJobError:
# Delete jobs that aren't there from scheduler
self.cancel(job_id)
return jobs
def test_convert_between_unix_times(self): # noqa
"""Can convert UNIX time to universal time and back."""
given_unix = 1328257004.456 # as returned by time.time()
self.assertEquals(times.to_unix(times.from_unix(given_unix)),
int(given_unix))
given_dt = self.sometime_univ
self.assertEquals(times.from_unix(times.to_unix(given_dt)), given_dt)
def is_modified_since(self, dt):
"""
Compares datetime `dt` with `If-Modified-Since` header value.
Returns True if `dt` is newer than `If-Modified-Since`, False otherwise.
"""
if_modified_since = parse_http_date('if-modified-since', self.headers)
if if_modified_since:
return times.to_unix(dt.replace(microsecond=0)) > times.to_unix(if_modified_since)
return True
def test_convert_between_unix_times(self): # noqa
"""Can convert UNIX time to universal time and back."""
given_unix = 1328257004.456 # as returned by time.time()
self.assertEquals(
times.to_unix(times.from_unix(given_unix)),
int(given_unix)
)
given_dt = self.sometime_univ
self.assertEquals(times.from_unix(times.to_unix(given_dt)),
given_dt)
def test_enqueue_in(self):
"""
Ensure that jobs have the right scheduled time.
"""
right_now = times.now()
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertIn(job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(right_now + time_delta))
time_delta = timedelta(hours=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(right_now + time_delta))
def check_rate_limit(r):
"""
Check the rate limit and sleep off if it is hit.
r - The response object from requests.
"""
try:
# If we hit the rate limit sleep
remain = r.headers["x-rate-limit-remaining"]
remain = int(remain)
if remain <= RATE_LIMIT_BUFFER:
log.debug("Hit rate limit - {}", remain)
now = r.headers["date"]
now = times.parse(now)
now = times.to_unix(now)
reset = r.headers["x-rate-limit-reset"]
reset = int(reset)
# Sleep past the reset time
log.debug("Rate limit reset in {} seconds", reset - now)
sleep(reset - now + RESET_BUFFER)
except KeyError as e:
# We dont have the proper headers
log.error("Header not found - {}", e)
sleep(FAILURE_RETRY)
def enqueue_job(self, job):
"""
Move a scheduled job to a queue. In addition, it also does puts the job
back into the scheduler if needed.
"""
self.log.debug('Pushing {0} to {1}'.format(job.id, job.origin))
interval = job.meta.get('interval', None)
repeat = job.meta.get('repeat', None)
# If job is a repeated job, decrement counter
if repeat:
job.meta['repeat'] = int(repeat) - 1
job.enqueued_at = times.now()
job.save()
queue = self.get_queue_for_job(job)
queue.push_job_id(job.id)
self.connection.zrem(self.scheduled_jobs_key, job.id)
if interval:
# If this is a repeat job and counter has reached 0, don't repeat
if repeat is not None:
if job.meta['repeat'] == 0:
return
self.connection._zadd(self.scheduled_jobs_key,
times.to_unix(times.now()) + int(interval),
job.id)
def schedule(self,
scheduled_time,
func,
args=None,
kwargs=None,
interval=None,
repeat=None,
result_ttl=None):
"""
Schedule a job to be periodically executed, at a certain interval.
"""
# Set result_ttl to -1 for periodic jobs, if result_ttl not specified
if interval is not None and result_ttl is None:
result_ttl = -1
job = self._create_job(func,
args=args,
kwargs=kwargs,
commit=False,
result_ttl=result_ttl)
if interval is not None:
job.meta['interval'] = int(interval)
if repeat is not None:
job.meta['repeat'] = int(repeat)
if repeat and interval is None:
raise ValueError("Can't repeat a job without interval argument")
job.save()
self.connection._zadd(self.scheduled_jobs_key,
times.to_unix(scheduled_time), job.id)
return job
def enqueue_job(self, job):
"""
Move a scheduled job to a queue. In addition, it also does puts the job
back into the scheduler if needed.
"""
self.log.debug('Pushing {0} to {1}'.format(job.id, job.origin))
interval = job.meta.get('interval', None)
repeat = job.meta.get('repeat', None)
# If job is a repeated job, decrement counter
if repeat:
job.meta['repeat'] = int(repeat) - 1
job.enqueued_at = times.now()
job.save()
queue = self.get_queue_for_job(job)
queue.push_job_id(job.id)
self.connection.zrem(self.scheduled_jobs_key, job.id)
if interval:
# If this is a repeat job and counter has reached 0, don't repeat
if repeat is not None:
if job.meta['repeat'] == 0:
return
self.connection._zadd(self.scheduled_jobs_key,
times.to_unix(times.now()) + int(interval),
job.id)
def get_jobs_to_queue(self, with_times=False):
"""
Returns a list of job instances that should be queued
(score lower than current timestamp).
If with_times is True a list of tuples consisting of the job instance and
it's scheduled execution time is returned.
"""
return self.get_jobs(times.to_unix(times.now()), with_times=with_times)
def get_jobs_to_queue(self, with_times=False):
"""
Returns a list of job instances that should be queued
(score lower than current timestamp).
If with_times is True a list of tuples consisting of the job instance and
it's scheduled execution time is returned.
"""
return self.get_jobs(times.to_unix(times.now()), with_times=with_times)
def test_enqueue_in(self):
"""
Ensure that jobs have the right scheduled time.
"""
right_now = times.now()
time_delta = timedelta(minutes=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(right_now + time_delta))
time_delta = timedelta(hours=1)
job = self.scheduler.enqueue_in(time_delta, say_hello)
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(right_now + time_delta))
def enqueue_in(self, time_delta, func, *args, **kwargs):
"""
Similar to ``enqueue_at``, but accepts a timedelta instead of datetime object.
The job's scheduled execution time will be calculated by adding the timedelta
to times.now().
"""
job = self._create_job(func, args=args, kwargs=kwargs)
self.connection._zadd(self.scheduled_jobs_key,
times.to_unix(times.now() + time_delta), job.id)
return job
def test_change_execution_time(self):
"""
Ensure ``change_execution_time`` is called, ensure that job's score is updated
"""
job = self.scheduler.enqueue_at(times.now(), say_hello)
new_date = datetime(2010, 1, 1)
self.scheduler.change_execution_time(job, new_date)
self.assertEqual(times.to_unix(new_date),
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
self.scheduler.cancel(job)
self.assertRaises(ValueError, self.scheduler.change_execution_time, job, new_date)
def test_create_scheduled_job(self):
"""
Ensure that scheduled jobs are put in the scheduler queue with the right score
"""
scheduled_time = times.now()
job = self.scheduler.enqueue_at(scheduled_time, say_hello)
self.assertEqual(job, Job.fetch(job.id, connection=self.testconn))
self.assertIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(scheduled_time))
def test_job_with_intervals_get_rescheduled(self):
"""
Ensure jobs with interval attribute are put back in the scheduler
"""
time_now = times.now()
interval = 10
job = self.scheduler.schedule(time_now, say_hello, interval=interval)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(time_now) + interval)
# Now the same thing using enqueue_periodic
job = self.scheduler.enqueue_periodic(time_now, interval, None, say_hello)
self.scheduler.enqueue_job(job)
self.assertIn(job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(time_now) + interval)
def enqueue_in(self, time_delta, func, *args, **kwargs):
"""
Similar to ``enqueue_at``, but accepts a timedelta instead of datetime object.
The job's scheduled execution time will be calculated by adding the timedelta
to times.now().
"""
job = self._create_job(func, args=args, kwargs=kwargs)
self.connection._zadd(self.scheduled_jobs_key,
times.to_unix(times.now() + time_delta),
job.id)
return job
def test_change_execution_time(self):
"""
Ensure ``change_execution_time`` is called, ensure that job's score is updated
"""
job = self.scheduler.enqueue_at(times.now(), say_hello)
new_date = datetime(2010, 1, 1)
self.scheduler.change_execution_time(job, new_date)
self.assertEqual(
times.to_unix(new_date),
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id))
self.scheduler.cancel(job)
self.assertRaises(ValueError, self.scheduler.change_execution_time,
job, new_date)
def test_create_scheduled_job(self):
"""
Ensure that scheduled jobs are put in the scheduler queue with the right score
"""
scheduled_time = times.now()
job = self.scheduler.enqueue_at(scheduled_time, say_hello)
self.assertEqual(job, Job.fetch(job.id, connection=self.testconn))
self.assertIn(
job.id,
self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1))
self.assertEqual(
self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id),
times.to_unix(scheduled_time))
def get_jobs(self, until=None, with_times=False):
"""
Returns a list of job instances that will be queued until the given time.
If no 'until' argument is given all jobs are returned. This function
accepts datetime and timedelta instances as well as integers representing
epoch values.
If with_times is True a list of tuples consisting of the job instance and
it's scheduled execution time is returned.
"""
def epoch_to_datetime(epoch):
return datetime.fromtimestamp(float(epoch))
if until is None:
until = "+inf"
elif isinstance(until, datetime):
until = times.to_unix(until)
elif isinstance(until, timedelta):
until = times.to_unix((times.now() + until))
job_ids = self.connection.zrangebyscore(
self.scheduled_jobs_key,
0,
until,
withscores=with_times,
score_cast_func=epoch_to_datetime)
if not with_times:
job_ids = zip(job_ids, repeat(None))
jobs = []
for job_id, sched_time in job_ids:
try:
job = Job.fetch(job_id, connection=self.connection)
if with_times:
jobs.append((job, sched_time))
else:
jobs.append(job)
except NoSuchJobError:
# Delete jobs that aren't there from scheduler
self.cancel(job_id)
return jobs
def change_execution_time(self, job, date_time):
"""
Change a job's execution time. Wrap this in a transaction to prevent race condition.
"""
with self.connection._pipeline() as pipe:
while 1:
try:
pipe.watch(self.scheduled_jobs_key)
if pipe.zscore(self.scheduled_jobs_key, job.id) is None:
raise ValueError('Job not in scheduled jobs queue')
pipe.zadd(self.scheduled_jobs_key, times.to_unix(date_time), job.id)
break
except WatchError:
# If job is still in the queue, retry otherwise job is already executed
# so we raise an error
if pipe.zscore(self.scheduled_jobs_key, job.id) is None:
raise ValueError('Job not in scheduled jobs queue')
continue
def change_execution_time(self, job, date_time):
"""
Change a job's execution time. Wrap this in a transaction to prevent race condition.
"""
with self.connection._pipeline() as pipe:
while 1:
try:
pipe.watch(self.scheduled_jobs_key)
if pipe.zscore(self.scheduled_jobs_key, job.id) is None:
raise ValueError('Job not in scheduled jobs queue')
pipe.zadd(self.scheduled_jobs_key,
times.to_unix(date_time), job.id)
break
except WatchError:
# If job is still in the queue, retry otherwise job is already executed
# so we raise an error
if pipe.zscore(self.scheduled_jobs_key, job.id) is None:
raise ValueError('Job not in scheduled jobs queue')
continue
def rest_rate_limit(r):
"""
Check the rate limit and sleep it off if hit.
"""
try:
#limit = int(r.headers["X-Rate-Limit-Limit"])
remain = int(r.headers["X-Rate-Limit-Remaining"])
reset = int(r.headers["X-Rate-Limit-Reset"])
curtime = times.to_unix(times.parse(r.headers["date"]))
except KeyError as e:
# We dont have the proper headers
log.error("Header not found - {}", e)
sleep(RETRY_AFTER)
return
if remain <= RATE_LIMIT_BUFFER:
log.debug("Hit rate limit - {}", remain)
log.debug("Rate limit reset in {} seconds", reset - curtime)
sleep(reset - curtime + RESET_BUFFER)
def schedule(self, scheduled_time, func, args=None, kwargs=None,
interval=None, repeat=None, result_ttl=None):
"""
Schedule a job to be periodically executed, at a certain interval.
"""
# Set result_ttl to -1 for periodic jobs, if result_ttl not specified
if interval is not None and result_ttl is None:
result_ttl = -1
job = self._create_job(func, args=args, kwargs=kwargs, commit=False,
result_ttl=result_ttl)
if interval is not None:
job.meta['interval'] = int(interval)
if repeat is not None:
job.meta['repeat'] = int(repeat)
if repeat and interval is None:
raise ValueError("Can't repeat a job without interval argument")
job.save()
self.connection._zadd(self.scheduled_jobs_key,
times.to_unix(scheduled_time),
job.id)
return job
def enqueue_at(self, scheduled_time, func, *args, **kwargs):
"""
Pushes a job to the scheduler queue. The scheduled queue is a Redis sorted
set ordered by timestamp - which in this case is job's scheduled execution time.
Usage:
from datetime import datetime
from redis import Redis
from rq.scheduler import Scheduler
from foo import func
redis = Redis()
scheduler = Scheduler(queue_name='default', connection=redis)
scheduler.enqueue_at(datetime(2020, 1, 1), func, 'argument', keyword='argument')
"""
job = self._create_job(func, args=args, kwargs=kwargs)
self.connection._zadd(self.scheduled_jobs_key,
times.to_unix(scheduled_time), job.id)
return job
def enqueue_at(self, scheduled_time, func, *args, **kwargs):
"""
Pushes a job to the scheduler queue. The scheduled queue is a Redis sorted
set ordered by timestamp - which in this case is job's scheduled execution time.
Usage:
from datetime import datetime
from redis import Redis
from rq.scheduler import Scheduler
from foo import func
redis = Redis()
scheduler = Scheduler(queue_name='default', connection=redis)
scheduler.enqueue_at(datetime(2020, 1, 1), func, 'argument', keyword='argument')
"""
job = self._create_job(func, args=args, kwargs=kwargs)
self.connection._zadd(self.scheduled_jobs_key,
times.to_unix(scheduled_time),
job.id)
return job
def set_last_modified(self, dt):
if dt:
self.headers['Last-Modified'] = http_date(times.to_unix(dt))
else:
self.headers.pop('Last-Modified', None)
def test_convert_non_numeric_to_unix(self):
"""to_unix refuses to accept non-numeric input"""
with self.assertRaises(TypeError):
times.to_unix('lol')
def test_convert_datetime_to_unix_time(self): # noqa
"""Can convert UNIX time to universal time."""
self.assertEquals(
times.to_unix(datetime(2012, 2, 3, 8, 16, 44)),
1328257004.0
)
def df(dt):
"""Format a datetime to something JS likes"""
return 1000 * times.to_unix(dt)
def test_convert_datetime_to_unix_time(self): # noqa
"""Can convert UNIX time to universal time."""
self.assertEquals(
times.to_unix(datetime(2012, 2, 3, 8, 16, 44)),
1328257004.0
)
def test_convert_non_numeric_to_unix(self):
"""to_unix refuses to accept non-numeric input"""
with self.assertRaises(TypeError):
times.to_unix('lol')
