def test_job_with_intervals_get_rescheduled(self): """ Ensure jobs with interval attribute are put back in the scheduler """ time_now = times.now() interval = 10 job = self.scheduler.schedule(time_now, say_hello, interval=interval) self.scheduler.enqueue_job(job) self.assertIn( job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)) self.assertEqual( self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(time_now) + interval) # Now the same thing using enqueue_periodic job = self.scheduler.enqueue_periodic(time_now, interval, None, say_hello) self.scheduler.enqueue_job(job) self.assertIn( job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)) self.assertEqual( self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(time_now) + interval)
def get_jobs(self, until=None, with_times=False): """ Returns a list of job instances that will be queued until the given time. If no 'until' argument is given all jobs are returned. This function accepts datetime and timedelta instances as well as integers representing epoch values. If with_times is True a list of tuples consisting of the job instance and it's scheduled execution time is returned. """ def epoch_to_datetime(epoch): return datetime.fromtimestamp(float(epoch)) if until is None: until = "+inf" elif isinstance(until, datetime): until = times.to_unix(until) elif isinstance(until, timedelta): until = times.to_unix((times.now() + until)) job_ids = self.connection.zrangebyscore(self.scheduled_jobs_key, 0, until, withscores=with_times, score_cast_func=epoch_to_datetime) if not with_times: job_ids = zip(job_ids, repeat(None)) jobs = [] for job_id, sched_time in job_ids: try: job = Job.fetch(job_id, connection=self.connection) if with_times: jobs.append((job, sched_time)) else: jobs.append(job) except NoSuchJobError: # Delete jobs that aren't there from scheduler self.cancel(job_id) return jobs
def test_convert_between_unix_times(self): # noqa """Can convert UNIX time to universal time and back.""" given_unix = 1328257004.456 # as returned by time.time() self.assertEquals(times.to_unix(times.from_unix(given_unix)), int(given_unix)) given_dt = self.sometime_univ self.assertEquals(times.from_unix(times.to_unix(given_dt)), given_dt)
def is_modified_since(self, dt): """ Compares datetime `dt` with `If-Modified-Since` header value. Returns True if `dt` is newer than `If-Modified-Since`, False otherwise. """ if_modified_since = parse_http_date('if-modified-since', self.headers) if if_modified_since: return times.to_unix(dt.replace(microsecond=0)) > times.to_unix(if_modified_since) return True
def test_convert_between_unix_times(self): # noqa """Can convert UNIX time to universal time and back.""" given_unix = 1328257004.456 # as returned by time.time() self.assertEquals( times.to_unix(times.from_unix(given_unix)), int(given_unix) ) given_dt = self.sometime_univ self.assertEquals(times.from_unix(times.to_unix(given_dt)), given_dt)
def test_enqueue_in(self): """ Ensure that jobs have the right scheduled time. """ right_now = times.now() time_delta = timedelta(minutes=1) job = self.scheduler.enqueue_in(time_delta, say_hello) self.assertIn(job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)) self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(right_now + time_delta)) time_delta = timedelta(hours=1) job = self.scheduler.enqueue_in(time_delta, say_hello) self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(right_now + time_delta))
def check_rate_limit(r): """ Check the rate limit and sleep off if it is hit. r - The response object from requests. """ try: # If we hit the rate limit sleep remain = r.headers["x-rate-limit-remaining"] remain = int(remain) if remain <= RATE_LIMIT_BUFFER: log.debug("Hit rate limit - {}", remain) now = r.headers["date"] now = times.parse(now) now = times.to_unix(now) reset = r.headers["x-rate-limit-reset"] reset = int(reset) # Sleep past the reset time log.debug("Rate limit reset in {} seconds", reset - now) sleep(reset - now + RESET_BUFFER) except KeyError as e: # We dont have the proper headers log.error("Header not found - {}", e) sleep(FAILURE_RETRY)
def enqueue_job(self, job): """ Move a scheduled job to a queue. In addition, it also does puts the job back into the scheduler if needed. """ self.log.debug('Pushing {0} to {1}'.format(job.id, job.origin)) interval = job.meta.get('interval', None) repeat = job.meta.get('repeat', None) # If job is a repeated job, decrement counter if repeat: job.meta['repeat'] = int(repeat) - 1 job.enqueued_at = times.now() job.save() queue = self.get_queue_for_job(job) queue.push_job_id(job.id) self.connection.zrem(self.scheduled_jobs_key, job.id) if interval: # If this is a repeat job and counter has reached 0, don't repeat if repeat is not None: if job.meta['repeat'] == 0: return self.connection._zadd(self.scheduled_jobs_key, times.to_unix(times.now()) + int(interval), job.id)
def schedule(self, scheduled_time, func, args=None, kwargs=None, interval=None, repeat=None, result_ttl=None): """ Schedule a job to be periodically executed, at a certain interval. """ # Set result_ttl to -1 for periodic jobs, if result_ttl not specified if interval is not None and result_ttl is None: result_ttl = -1 job = self._create_job(func, args=args, kwargs=kwargs, commit=False, result_ttl=result_ttl) if interval is not None: job.meta['interval'] = int(interval) if repeat is not None: job.meta['repeat'] = int(repeat) if repeat and interval is None: raise ValueError("Can't repeat a job without interval argument") job.save() self.connection._zadd(self.scheduled_jobs_key, times.to_unix(scheduled_time), job.id) return job
def get_jobs_to_queue(self, with_times=False): """ Returns a list of job instances that should be queued (score lower than current timestamp). If with_times is True a list of tuples consisting of the job instance and it's scheduled execution time is returned. """ return self.get_jobs(times.to_unix(times.now()), with_times=with_times)
def test_enqueue_in(self): """ Ensure that jobs have the right scheduled time. """ right_now = times.now() time_delta = timedelta(minutes=1) job = self.scheduler.enqueue_in(time_delta, say_hello) self.assertIn( job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)) self.assertEqual( self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(right_now + time_delta)) time_delta = timedelta(hours=1) job = self.scheduler.enqueue_in(time_delta, say_hello) self.assertEqual( self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(right_now + time_delta))
def enqueue_in(self, time_delta, func, *args, **kwargs): """ Similar to ``enqueue_at``, but accepts a timedelta instead of datetime object. The job's scheduled execution time will be calculated by adding the timedelta to times.now(). """ job = self._create_job(func, args=args, kwargs=kwargs) self.connection._zadd(self.scheduled_jobs_key, times.to_unix(times.now() + time_delta), job.id) return job
def test_change_execution_time(self): """ Ensure ``change_execution_time`` is called, ensure that job's score is updated """ job = self.scheduler.enqueue_at(times.now(), say_hello) new_date = datetime(2010, 1, 1) self.scheduler.change_execution_time(job, new_date) self.assertEqual(times.to_unix(new_date), self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id)) self.scheduler.cancel(job) self.assertRaises(ValueError, self.scheduler.change_execution_time, job, new_date)
def test_create_scheduled_job(self): """ Ensure that scheduled jobs are put in the scheduler queue with the right score """ scheduled_time = times.now() job = self.scheduler.enqueue_at(scheduled_time, say_hello) self.assertEqual(job, Job.fetch(job.id, connection=self.testconn)) self.assertIn(job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)) self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(scheduled_time))
def test_job_with_intervals_get_rescheduled(self): """ Ensure jobs with interval attribute are put back in the scheduler """ time_now = times.now() interval = 10 job = self.scheduler.schedule(time_now, say_hello, interval=interval) self.scheduler.enqueue_job(job) self.assertIn(job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)) self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(time_now) + interval) # Now the same thing using enqueue_periodic job = self.scheduler.enqueue_periodic(time_now, interval, None, say_hello) self.scheduler.enqueue_job(job) self.assertIn(job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)) self.assertEqual(self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(time_now) + interval)
def test_change_execution_time(self): """ Ensure ``change_execution_time`` is called, ensure that job's score is updated """ job = self.scheduler.enqueue_at(times.now(), say_hello) new_date = datetime(2010, 1, 1) self.scheduler.change_execution_time(job, new_date) self.assertEqual( times.to_unix(new_date), self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id)) self.scheduler.cancel(job) self.assertRaises(ValueError, self.scheduler.change_execution_time, job, new_date)
def test_create_scheduled_job(self): """ Ensure that scheduled jobs are put in the scheduler queue with the right score """ scheduled_time = times.now() job = self.scheduler.enqueue_at(scheduled_time, say_hello) self.assertEqual(job, Job.fetch(job.id, connection=self.testconn)) self.assertIn( job.id, self.testconn.zrange(self.scheduler.scheduled_jobs_key, 0, 1)) self.assertEqual( self.testconn.zscore(self.scheduler.scheduled_jobs_key, job.id), times.to_unix(scheduled_time))
def get_jobs(self, until=None, with_times=False): """ Returns a list of job instances that will be queued until the given time. If no 'until' argument is given all jobs are returned. This function accepts datetime and timedelta instances as well as integers representing epoch values. If with_times is True a list of tuples consisting of the job instance and it's scheduled execution time is returned. """ def epoch_to_datetime(epoch): return datetime.fromtimestamp(float(epoch)) if until is None: until = "+inf" elif isinstance(until, datetime): until = times.to_unix(until) elif isinstance(until, timedelta): until = times.to_unix((times.now() + until)) job_ids = self.connection.zrangebyscore( self.scheduled_jobs_key, 0, until, withscores=with_times, score_cast_func=epoch_to_datetime) if not with_times: job_ids = zip(job_ids, repeat(None)) jobs = [] for job_id, sched_time in job_ids: try: job = Job.fetch(job_id, connection=self.connection) if with_times: jobs.append((job, sched_time)) else: jobs.append(job) except NoSuchJobError: # Delete jobs that aren't there from scheduler self.cancel(job_id) return jobs
def change_execution_time(self, job, date_time): """ Change a job's execution time. Wrap this in a transaction to prevent race condition. """ with self.connection._pipeline() as pipe: while 1: try: pipe.watch(self.scheduled_jobs_key) if pipe.zscore(self.scheduled_jobs_key, job.id) is None: raise ValueError('Job not in scheduled jobs queue') pipe.zadd(self.scheduled_jobs_key, times.to_unix(date_time), job.id) break except WatchError: # If job is still in the queue, retry otherwise job is already executed # so we raise an error if pipe.zscore(self.scheduled_jobs_key, job.id) is None: raise ValueError('Job not in scheduled jobs queue') continue
def rest_rate_limit(r): """ Check the rate limit and sleep it off if hit. """ try: #limit = int(r.headers["X-Rate-Limit-Limit"]) remain = int(r.headers["X-Rate-Limit-Remaining"]) reset = int(r.headers["X-Rate-Limit-Reset"]) curtime = times.to_unix(times.parse(r.headers["date"])) except KeyError as e: # We dont have the proper headers log.error("Header not found - {}", e) sleep(RETRY_AFTER) return if remain <= RATE_LIMIT_BUFFER: log.debug("Hit rate limit - {}", remain) log.debug("Rate limit reset in {} seconds", reset - curtime) sleep(reset - curtime + RESET_BUFFER)
def enqueue_at(self, scheduled_time, func, *args, **kwargs): """ Pushes a job to the scheduler queue. The scheduled queue is a Redis sorted set ordered by timestamp - which in this case is job's scheduled execution time. Usage: from datetime import datetime from redis import Redis from rq.scheduler import Scheduler from foo import func redis = Redis() scheduler = Scheduler(queue_name='default', connection=redis) scheduler.enqueue_at(datetime(2020, 1, 1), func, 'argument', keyword='argument') """ job = self._create_job(func, args=args, kwargs=kwargs) self.connection._zadd(self.scheduled_jobs_key, times.to_unix(scheduled_time), job.id) return job
def set_last_modified(self, dt): if dt: self.headers['Last-Modified'] = http_date(times.to_unix(dt)) else: self.headers.pop('Last-Modified', None)
def test_convert_non_numeric_to_unix(self): """to_unix refuses to accept non-numeric input""" with self.assertRaises(TypeError): times.to_unix('lol')
def test_convert_datetime_to_unix_time(self): # noqa """Can convert UNIX time to universal time.""" self.assertEquals( times.to_unix(datetime(2012, 2, 3, 8, 16, 44)), 1328257004.0 )
def df(dt): """Format a datetime to something JS likes""" return 1000 * times.to_unix(dt)