def test_job_data(self): schedule_job('fòo', schedule_in=-1, other_arg='lol') self.assertEqual(list(pending_jobs()), [{ 'id': 'fòo', 'other_arg': 'lol' }]) self.assertEqual(len(list(pending_jobs())), 0)
def test_legacy_redis(self): connection = redis.Redis(**REDIS) for i in range(10): schedule_job('foo{0}'.format(i), schedule_in=-1, connection=connection) jobs = list(pending_jobs(connection=connection, reschedule_in=-1)) self.assertEqual(len(jobs), 10) jobs = list(pending_jobs(connection=r, reschedule_in=-1)) self.assertEqual(len(jobs), 10)
def test_remove_keys(self): schedule_job('foobar', schedule_in=-1, attr='stuff', other=12, thing='blah blah') jobs = list(pending_jobs()) self.assertEqual(jobs, [{'id': 'foobar', 'attr': 'stuff', 'other': 12, 'thing': 'blah blah'}]) schedule_job('foobar', schedule_in=-1, attr=None, other=None, thing='blah blah') jobs = list(pending_jobs()) self.assertEqual(jobs, [{'id': 'foobar', 'thing': 'blah blah'}])
def handle_sentry(self, *args, **kwargs): if args: pk = args[0] feed = UniqueFeed.objects.get(pk=pk) return update_feed( feed.url, etag=feed.etag, modified=feed.modified, subscribers=feed.subscribers, request_timeout=feed.request_timeout, backoff_factor=feed.backoff_factor, error=feed.error, link=feed.link, title=feed.title, hub=feed.hub, ) ratio = UniqueFeed.UPDATE_PERIOD // 5 limit = max( 1, UniqueFeed.objects.filter(muted=False).count() // ratio) * 2 # Avoid queueing if the default or store queue is already full conn = redis.Redis(**settings.REDIS) for name in ['default', 'store']: queue = Queue(name=name, connection=conn) if queue.count > limit: logger.info( "{0} queue longer than limit, skipping update " "({1} > {2})".format(name, queue.count, limit)) jobs = pending_jobs(limit=limit, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60) for job in jobs: url = job.pop('id') job.pop('last_update', None) enqueue(update_feed, args=[url], kwargs=job, timeout=UniqueFeed.TIMEOUT_BASE * job.get( 'backoff_factor', 1))
def test_schedule_limit_items_count(self): for i in range(100): schedule_job('foo{0}'.format(i), schedule_in=-1) jobs = list(pending_jobs(limit=10)) self.assertEqual(len(jobs), 10) self.assertEqual(len(list(scheduled_jobs())), 90)
def handle_sentry(self, *args, **kwargs): if args: pk = args[0] feed = UniqueFeed.objects.get(pk=pk) data = feed.job_details return update_feed( feed.url, etag=data.get("etag"), modified=data.get("modified"), subscribers=data["subscribers"], backoff_factor=data["backoff_factor"], error=data.get("error"), link=data.get("link"), title=data.get("title"), hub=data.get("hub"), ) ratio = UniqueFeed.UPDATE_PERIOD // 5 limit = max(1, UniqueFeed.objects.filter(muted=False).count() // ratio) * 2 # Avoid queueing if the default or store queue is already full conn = get_redis_connection() for name in ["default", "store"]: queue = Queue(name=name, connection=conn) if queue.count > limit: logger.info( "{0} queue longer than limit, skipping update " "({1} > {2})".format(name, queue.count, limit) ) return jobs = pending_jobs(limit=limit, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection()) for job in jobs: url = job.pop("id") job.pop("last_update", None) enqueue(update_feed, args=[url], kwargs=job, timeout=UniqueFeed.TIMEOUT_BASE * job.get("backoff_factor", 1))
def test_reschedule(self): schedule_job('baz', schedule_in=-1) schedule_job('foo', schedule_in=10) jobs = list(pending_jobs(reschedule_in=20)) self.assertEqual(jobs, [{'id': 'baz'}]) schedule = list(scheduled_jobs(with_times=True)) foo = schedule[0] baz = schedule[1] self.assertEqual(foo[0], ('foo')) self.assertEqual(baz[0], ('baz')) self.assertEqual(foo[1] + 10, baz[1])
def test_remove_keys(self): schedule_job('foobar', schedule_in=-1, attr='stuff', other=12, thing='blah blah') jobs = list(pending_jobs()) self.assertEqual(jobs, [{ 'id': 'foobar', 'attr': 'stuff', 'other': 12, 'thing': 'blah blah' }]) schedule_job('foobar', schedule_in=-1, attr=None, other=None, thing='blah blah') jobs = list(pending_jobs()) self.assertEqual(jobs, [{'id': 'foobar', 'thing': 'blah blah'}])
def handle_sentry(self, *args, **kwargs): if args: pk = args[0] feed = UniqueFeed.objects.get(pk=pk) data = feed.job_details return update_feed( feed.url, etag=data.get('etag'), modified=data.get('modified'), subscribers=data.get('subscribers', 1), backoff_factor=data['backoff_factor'], error=data.get('error'), link=data.get('link'), title=data.get('title'), hub=data.get('hub'), ) ratio = UniqueFeed.UPDATE_PERIOD // 5 limit = max( 1, UniqueFeed.objects.filter(muted=False).count() // ratio) * 2 # Avoid queueing if the default or store queue is already full conn = get_redis_connection() for name in ['default', 'store']: queue = Queue(name=name, connection=conn) if queue.count > limit: logger.info("queue longer than limit, skipping update", queue=name, count=queue.count, limit=limit) return jobs = pending_jobs(limit=limit, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection()) for job in jobs: url = job.pop('id') job.pop('last_update', None) enqueue(update_feed, args=[url], kwargs=job, timeout=UniqueFeed.TIMEOUT_BASE * job.get('backoff_factor', 1))
def handle_sentry(self, *args, **kwargs): if args: pk = args[0] feed = UniqueFeed.objects.get(pk=pk) data = feed.job_details return update_feed( feed.url, etag=data.get('etag'), modified=data.get('modified'), subscribers=data.get('subscribers', 1), backoff_factor=data['backoff_factor'], error=data.get('error'), link=data.get('link'), title=data.get('title'), hub=data.get('hub'), ) ratio = UniqueFeed.UPDATE_PERIOD // 5 limit = max( 1, UniqueFeed.objects.filter(muted=False).count() // ratio) * 2 # Avoid queueing if the default or store queue is already full conn = get_redis_connection() for name in ['default', 'store']: queue = Queue(name=name, connection=conn) if queue.count > limit: logger.info( "queue longer than limit, skipping update", queue=name, count=queue.count, limit=limit) return jobs = pending_jobs(limit=limit, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection()) for job in jobs: url = job.pop('id') job.pop('last_update', None) enqueue(update_feed, args=[url], kwargs=job, timeout=UniqueFeed.TIMEOUT_BASE * job.get( 'backoff_factor', 1))
def test_update_feeds(self): u = UniqueFeed.objects.create( url='http://example.com/feed0', ) u.schedule() patch_job( u.url, last_update=(timezone.now() - timedelta(hours=1)).strftime('%s') ) u.schedule() UniqueFeed.objects.create( url='http://example.com/feed1', ).schedule() with self.assertNumQueries(0): jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0]['id'], u.url) u.delete() delete_job(u.url, connection=get_redis_connection()) with self.assertNumQueries(0): urls = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(urls), 0) u = UniqueFeed.objects.create( url='http://example.com/backoff', ) u.schedule() patch_job( u.url, backoff_factor=10, last_update=(timezone.now() - timedelta(hours=28)).strftime('%s') ) u.schedule() with self.assertNumQueries(0): jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 0) patch_job(u.url, backoff_factor=9) u.schedule() with self.assertNumQueries(0): jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0]['id'], u.url) self.assertEqual( UniqueFeed.TIMEOUT_BASE * jobs[0]['backoff_factor'], 180) patch_job(u.url, last_update=int(time.time())) u.schedule() with self.assertNumQueries(0): jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 0) UniqueFeed.objects.create( url='http://example.com/lol', ) for u in UniqueFeed.objects.all(): patch_job(u.url, last_update=( timezone.now() - timedelta(hours=54)).strftime('%s')) # No subscribers -> deletion with self.assertNumQueries(2): call_command('delete_unsubscribed') self.assertEqual(UniqueFeed.objects.count(), 0) u = UniqueFeed.objects.create( url='http://example.com/foo', ) u.schedule() patch_job( u.url, last_update=(timezone.now() - timedelta(hours=2)).strftime('%s')) u.schedule() u = UniqueFeed.objects.create( url='http://example.com/bar', ) u.schedule() patch_job( u.url, last_update=(timezone.now() - timedelta(hours=2)).strftime('%s')) u.schedule() jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0]['id'], 'http://example.com/bar') self.assertEqual(jobs[1]['id'], 'http://example.com/foo')
def test_schedule_non_unicode_data(self): schedule_job('bad', schedule_in=-1, etag=b'2013/6/29 \xa4W\xa4\xc8 09:51:31') job = list(pending_jobs())[0] self.assertEqual(job['etag'], b'2013/6/29 \xa4W\xa4\xc8 09:51:31')
def test_ordering(self): schedule_job('foo', schedule_in=-1) schedule_job('bar', schedule_in=-2) jobs = list(pending_jobs()) self.assertEqual(jobs[0]['id'], 'bar') self.assertEqual(jobs[1]['id'], 'foo')
def test_custom_connection(self): for i in range(10): schedule_job('foo{0}'.format(i), schedule_in=-1, connection=r) jobs = list(pending_jobs(connection=r)) self.assertEqual(len(jobs), 10)
def test_schedule_in_future(self): schedule_job('lol', schedule_in=10) self.assertEqual(len(list(pending_jobs())), 0) delete_job('lol')
def test_reschedule_existing(self): schedule_job('lol', schedule_in=-1) schedule_job('lol', schedule_in=10) self.assertEqual(len(list(pending_jobs())), 0) schedule_job('lol', schedule_in=-1) self.assertEqual(len(list(pending_jobs())), 1)
def test_job_data(self): schedule_job('fòo', schedule_in=-1, other_arg='lol') self.assertEqual(list(pending_jobs()), [{'id': 'fòo', 'other_arg': 'lol'}]) self.assertEqual(len(list(pending_jobs())), 0)
def test_job_deletion(self): schedule_job('bar', schedule_in=-1) delete_job('bar') self.assertEqual(len(list(pending_jobs())), 0)
def test_update_feeds(self): u = UniqueFeed.objects.create( url='http://example.com/feed0', last_update=timezone.now() - timedelta(hours=1), ) u.schedule() UniqueFeed.objects.create( url='http://example.com/feed1', ).schedule() with self.assertNumQueries(0): jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60)) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0]['id'], u.url) u.delete() delete_job(u.url) with self.assertNumQueries(0): urls = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60)) self.assertEqual(len(urls), 0) u = UniqueFeed.objects.create( url='http://example.com/backoff', last_update=timezone.now() - timedelta(hours=28), backoff_factor=10, ) with self.assertNumQueries(0): jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60)) self.assertEqual(len(jobs), 0) u.backoff_factor = 9 u.save() u.schedule() with self.assertNumQueries(0): jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60)) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0]['id'], u.url) self.assertEqual( UniqueFeed.TIMEOUT_BASE * jobs[0]['backoff_factor'], 180) UniqueFeed.objects.update(last_update=timezone.now()) with self.assertNumQueries(0): jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60)) self.assertEqual(len(jobs), 0) UniqueFeed.objects.create( url='http://example.com/lol', ) UniqueFeed.objects.update( last_update=timezone.now() - timedelta(hours=54)) # No subscribers -> deletion with self.assertNumQueries(2): call_command('delete_unsubscribed') self.assertEqual(UniqueFeed.objects.count(), 0) UniqueFeed.objects.create( url='http://example.com/foo', last_update=timezone.now() - timedelta(hours=2), ).schedule() UniqueFeed.objects.create( url='http://example.com/bar', last_update=timezone.now() - timedelta(hours=2), last_loop=timezone.now() - timedelta(hours=2), ).schedule() jobs = list(pending_jobs( limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60)) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0]['id'], 'http://example.com/bar') self.assertEqual(jobs[1]['id'], 'http://example.com/foo')
def test_update_feeds(self): u = UniqueFeed.objects.create(url='http://example.com/feed0', ) u.schedule() patch_job(u.url, last_update=(timezone.now() - timedelta(hours=1)).strftime('%s')) u.schedule() UniqueFeed.objects.create(url='http://example.com/feed1', ).schedule() with self.assertNumQueries(0): jobs = list( pending_jobs(limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0]['id'], u.url) u.delete() delete_job(u.url, connection=get_redis_connection()) with self.assertNumQueries(0): urls = list( pending_jobs(limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(urls), 0) u = UniqueFeed.objects.create(url='http://example.com/backoff', ) u.schedule() patch_job(u.url, backoff_factor=10, last_update=(timezone.now() - timedelta(hours=28)).strftime('%s')) u.schedule() with self.assertNumQueries(0): jobs = list( pending_jobs(limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 0) patch_job(u.url, backoff_factor=9) u.schedule() with self.assertNumQueries(0): jobs = list( pending_jobs(limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0]['id'], u.url) self.assertEqual( UniqueFeed.TIMEOUT_BASE * jobs[0]['backoff_factor'], 180) patch_job(u.url, last_update=int(time.time())) u.schedule() with self.assertNumQueries(0): jobs = list( pending_jobs(limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 0) UniqueFeed.objects.create(url='http://example.com/lol', ) for u in UniqueFeed.objects.all(): patch_job(u.url, last_update=(timezone.now() - timedelta(hours=54)).strftime('%s')) # No subscribers -> deletion with self.assertNumQueries(2): call_command('delete_unsubscribed') self.assertEqual(UniqueFeed.objects.count(), 0) u = UniqueFeed.objects.create(url='http://example.com/foo', ) u.schedule() patch_job(u.url, last_update=(timezone.now() - timedelta(hours=2)).strftime('%s')) u.schedule() u = UniqueFeed.objects.create(url='http://example.com/bar', ) u.schedule() patch_job(u.url, last_update=(timezone.now() - timedelta(hours=2)).strftime('%s')) u.schedule() jobs = list( pending_jobs(limit=5, reschedule_in=UniqueFeed.UPDATE_PERIOD * 60, connection=get_redis_connection())) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0]['id'], 'http://example.com/bar') self.assertEqual(jobs[1]['id'], 'http://example.com/foo')