def scheduler(): """Run rq-scheduler""" redis_client = get_rq_redis_client() scheduler = Scheduler(connection=redis_client) # Create the RediSearch index and begin indexing immediately. # If a previous index exists, delete it. tasks.index(config.sites, rebuild_index=True) # Schedule an indexing job to run every 30 minutes. # # This performs an update-in-place using the existing RediSearch index. # # TODO: We currently don't try to detect if we have outdated content in # the index -- i.e. when we reindexed a site, a URL was leftover in the # index that we didn't find on this round of indexing. # # NOTE: We need to define this here, at the time we run this command, # because there is no deduplication in the cron() method, and this app has # no "exactly once" startup/initialization step that we could use to call # code only once. scheduler.cron( "*/60 * * * *", func=tasks.index, args=[config.sites, False], use_local_timezone=True, timeout=tasks.INDEXING_TIMEOUT ) scheduler.run()
def main(): parser = argparse.ArgumentParser(description='Runs RQ scheduler') parser.add_argument('-H', '--host', default=os.environ.get('RQ_REDIS_HOST', 'localhost'), help="Redis host") parser.add_argument('-p', '--port', default=int(os.environ.get('RQ_REDIS_PORT', 6379)), type=int, help="Redis port number") parser.add_argument('-d', '--db', default=int(os.environ.get('RQ_REDIS_DB', 0)), type=int, help="Redis database") parser.add_argument('-P', '--password', default=os.environ.get('RQ_REDIS_PASSWORD'), help="Redis password") parser.add_argument('--url', '-u', default=os.environ.get('RQ_REDIS_URL') , help='URL describing Redis connection details. \ Overrides other connection arguments if supplied.') parser.add_argument('-i', '--interval', default=60, type=int , help="How often the scheduler checks for new jobs to add to the \ queue (in seconds).") parser.add_argument('--path', default='.', help='Specify the import path.') parser.add_argument('--pid', help='A filename to use for the PID file.', metavar='FILE') args = parser.parse_args() if args.path: sys.path = args.path.split(':') + sys.path if args.pid: pid = str(os.getpid()) filename = args.pid with open(filename, 'w') as f: f.write(pid) if args.url is not None: connection = Redis.from_url(args.url) else: connection = Redis(args.host, args.port, args.db, args.password) scheduler = Scheduler(connection=connection, interval=args.interval) scheduler.run()
def main(): parser = argparse.ArgumentParser(description='Runs RQ scheduler') parser.add_argument('-H', '--host', default='localhost', help="Redis host") parser.add_argument('-p', '--port', default=6379, type=int, help="Redis port number") parser.add_argument('-d', '--db', default=0, type=int, help="Redis database") parser.add_argument('-P', '--password', default=None, help="Redis password") parser.add_argument( '-i', '--interval', default=60, type=int, help="How often the scheduler checks for new jobs to add to the \ queue (in seconds).") args = parser.parse_args() connection = Redis(args.host, args.port, args.db, args.password) scheduler = Scheduler(connection=connection, interval=args.interval) scheduler.run()
def test_birth_and_death_registration(self): """ When scheduler registers it's birth, besides creating a key, it should also set an expiry that's a few seconds longer than it's polling interval so it automatically expires if scheduler is unexpectedly terminated. """ key = Scheduler.scheduler_key self.assertNotIn(key, tl(self.testconn.keys('*'))) scheduler = Scheduler(connection=self.testconn, interval=20) scheduler.register_birth() self.assertIn(key, tl(self.testconn.keys('*'))) self.assertEqual(self.testconn.ttl(key), 30) self.assertFalse(self.testconn.hexists(key, 'death')) self.assertRaises(ValueError, scheduler.register_birth) scheduler.register_death() self.assertTrue(self.testconn.hexists(key, 'death'))
def test_enqueue_job(self): """ When scheduled job is enqueued, make sure: - Job is removed from the sorted set of scheduled jobs - "enqueued_at" attribute is properly set - Job appears in the right queue """ now = datetime.utcnow() queue_name = 'foo' scheduler = Scheduler(connection=self.testconn, queue_name=queue_name) job = scheduler.enqueue_at(now, say_hello) self.scheduler.enqueue_job(job) self.assertNotIn(job, tl(self.testconn.zrange(scheduler.scheduled_jobs_key, 0, 10))) job = Job.fetch(job.id, connection=self.testconn) self.assertTrue(job.enqueued_at is not None) queue = scheduler.get_queue_for_job(job) self.assertIn(job, queue.jobs) queue = Queue.from_queue_key('rq:queue:{0}'.format(queue_name)) self.assertIn(job, queue.jobs)
def run_scheduler(): conn_kwargs = { 'db': app_settings.config.get('REDIS_DB') or 0, 'password': app_settings.config.get('REDIS_PWD') } if all( app_settings.config.get(attr) for attr in ['REDIS_MASTER_DNS', 'REDIS_PORT']): master = StrictRedis(host=app_settings.config['REDIS_MASTER_DNS'], port=app_settings.config['REDIS_PORT'], **conn_kwargs) else: sentinel = Sentinel(app_settings.config['REDIS_SENTINEL']) master = sentinel.master_for(app_settings.config['REDIS_MASTER'], **conn_kwargs) scheduler = Scheduler(connection=master) while True: try: scheduler.run() except ValueError: sleep(600)
def main(): parser = argparse.ArgumentParser(description='Runs RQ scheduler') parser.add_argument('-H', '--host', default='localhost', help="Redis host") parser.add_argument('-p', '--port', default=6379, type=int, help="Redis port number") parser.add_argument('-d', '--db', default=0, type=int, help="Redis database") parser.add_argument('-P', '--password', default=None, help="Redis password") args = parser.parse_args() connection = Redis(args.host, args.port, args.db, args.password) scheduler = Scheduler(connection=connection) scheduler.run()
def main(): parser = argparse.ArgumentParser(description='Runs RQ scheduler') parser.add_argument('-H', '--host', default='localhost', help="Redis host") parser.add_argument('-p', '--port', default=6379, type=int, help="Redis port number") parser.add_argument('-d', '--db', default=0, type=int, help="Redis database") parser.add_argument('-P', '--password', default=None, help="Redis password") parser.add_argument('--url', '-u', default=None , help='URL describing Redis connection details. \ Overrides other connection arguments if supplied.') parser.add_argument('-i', '--interval', default=60, type=int , help="How often the scheduler checks for new jobs to add to the \ queue (in seconds).") parser.add_argument('--path', default='.', help='Specify the import path.') args = parser.parse_args() if args.path: sys.path = args.path.split(':') + sys.path if args.url is not None: connection = Redis.from_url(args.url) else: connection = Redis(args.host, args.port, args.db, args.password) scheduler = Scheduler(connection=connection, interval=args.interval) scheduler.run()
def run_scheduler(): """Initializes a rq scheduler.""" conn = Redis( host=app.config['RQ_DEFAULT_HOST'], port=app.config['RQ_DEFAULT_PORT'], db=0, password=app.config['RQ_DEFAULT_PASSWORD'] ) setup_loghandlers('INFO') scheduler = Scheduler(connection=conn, interval=60.0) for _ in range(10): try: scheduler.run() except ValueError as exc: if exc.message == 'There\'s already an active RQ scheduler': scheduler.log.info( 'An RQ scheduler instance is already running. Retrying in ' '%d seconds.', 10, ) time.sleep(10) else: raise exc
def setUp(self): super(TestScheduler, self).setUp() self.scheduler = Scheduler(connection=self.testconn)
def main(): parser = argparse.ArgumentParser(description='Runs RQ scheduler') parser.add_argument('-H', '--host', default=os.environ.get('RQ_REDIS_HOST', 'localhost'), help="Redis host") parser.add_argument('-p', '--port', default=int(os.environ.get('RQ_REDIS_PORT', 6379)), type=int, help="Redis port number") parser.add_argument('-d', '--db', default=int(os.environ.get('RQ_REDIS_DB', 0)), type=int, help="Redis database") parser.add_argument('-P', '--password', default=os.environ.get('RQ_REDIS_PASSWORD'), help="Redis password") parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Show more output') parser.add_argument('--url', '-u', default=os.environ.get('RQ_REDIS_URL'), help='URL describing Redis connection details. \ Overrides other connection arguments if supplied.') parser.add_argument( '-i', '--interval', default=60.0, type=float, help="How often the scheduler checks for new jobs to add to the \ queue (in seconds, can be floating-point for more precision).") parser.add_argument('--path', default='.', help='Specify the import path.') parser.add_argument('--pid', help='A filename to use for the PID file.', metavar='FILE') args = parser.parse_args() if args.path: sys.path = args.path.split(':') + sys.path if args.pid: pid = str(os.getpid()) filename = args.pid with open(filename, 'w') as f: f.write(pid) if args.url is not None: connection = Redis.from_url(args.url) elif os.getenv('REDISTOGO_URL'): redis_url = os.getenv('REDISTOGO_URL') if not redis_url: raise RuntimeError('Set up Redis To Go first.') urlparse.uses_netloc.append('redis') url = urlparse.urlparse(redis_url) connection = Redis(host=url.hostname, port=url.port, db=0, password=url.password) elif args.host is not None: connection = Redis(args.host, args.port, args.db, args.password) else: connection = Redis() if args.verbose: level = 'DEBUG' else: level = 'INFO' setup_loghandlers(level) scheduler = Scheduler(connection=connection, interval=args.interval) scheduler.run()
import app.logging from app.rich_menu import RichMenu from app.scheduling import ReminderJob, ReminderWorker from rasa.lineagent import LineAgent from rasa.store import scheduler_store, tracker_store logger = logging.getLogger(__name__) logger.debug("Starting worker") line_access_token = os.getenv('LINE_CHANNEL_ACCESS_TOKEN', None) rich_menu = RichMenu(line_access_token) rich_menu.setup() agent = LineAgent.load("models/dialogue", interpreter=RasaNLUInterpreter("models/current/nlu"), tracker_store=tracker_store) workerKwargs = {"rich_menu": rich_menu, "agent": agent} listen = ['high', 'default', 'low'] scheduler = Scheduler(connection=scheduler_store, interval=60, job_class=ReminderJob) Process(target=scheduler.run).start() with Connection(scheduler_store): worker = ReminderWorker(map(Queue, listen), job_class=ReminderJob) logger.info("Worker is ready.") worker.work(workerKwargs=workerKwargs)
def main(): parser = argparse.ArgumentParser(description='Runs RQ scheduler') parser.add_argument('-b', '--burst', action='store_true', default=False, help='Run in burst mode (quit after all work is done)') parser.add_argument('-H', '--host', default=os.environ.get('RQ_REDIS_HOST', 'localhost'), help="Redis host") parser.add_argument('-p', '--port', default=int(os.environ.get('RQ_REDIS_PORT', 6379)), type=int, help="Redis port number") parser.add_argument('-d', '--db', default=int(os.environ.get('RQ_REDIS_DB', 0)), type=int, help="Redis database") parser.add_argument('-P', '--password', default=os.environ.get('RQ_REDIS_PASSWORD'), help="Redis password") parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Show more output') parser.add_argument('--quiet', '-q', action='store_true', default=False, help='Show less output') parser.add_argument('--url', '-u', default=os.environ.get('RQ_REDIS_URL'), help='URL describing Redis connection details. \ Overrides other connection arguments if supplied.') parser.add_argument( '-i', '--interval', default=60.0, type=float, help="How often the scheduler checks for new jobs to add to the \ queue (in seconds, can be floating-point for more precision).") parser.add_argument('--path', default='.', help='Specify the import path.') parser.add_argument('--pid', help='A filename to use for the PID file.', metavar='FILE') parser.add_argument('-j', '--job-class', help='Custom RQ Job class') parser.add_argument('-q', '--queue-class', help='Custom RQ Queue class') args = parser.parse_args() if args.path: sys.path = args.path.split(':') + sys.path if args.pid: pid = str(os.getpid()) filename = args.pid with open(filename, 'w') as f: f.write(pid) if args.url is not None: connection = Redis.from_url(args.url) else: connection = Redis(args.host, args.port, args.db, args.password) if args.verbose: level = 'DEBUG' elif args.quiet: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) scheduler = Scheduler(connection=connection, interval=args.interval, job_class=args.job_class, queue_class=args.queue_class) scheduler.run(burst=args.burst)
def main(): conn = redis.from_url(os.getenv('REDISCLOUD_URL', 'redis://localhost:6379')) scheduler = Scheduler(connection=conn, interval=30) scheduler.run()
def scheduler(): """Run rq-scheduler""" redis_client = get_rq_redis_client() scheduler = Scheduler(connection=redis_client) schedule(scheduler, redis_client, config) scheduler.run()
def test_scheduler_w_o_explicit_connection(self): """ Ensure instantiating Scheduler w/o explicit connection works. """ s = Scheduler() self.assertEqual(s.connection, self.testconn)
# -*- coding: utf-8 -*- """ IncetOps.rqscheduler_worker ~~~~~~~~~~~~~~ The working process of the RQ-Scheduler queue. :copyright: (c) 2018 by staugur. :license: MIT, see LICENSE for more details. """ if __name__ == '__main__': import setproctitle from redis import from_url from config import GLOBAL, REDIS from rq_scheduler.scheduler import Scheduler setproctitle.setproctitle(GLOBAL['ProcessName'] + '.rqscheduler') scheduler = Scheduler(connection=from_url(REDIS), interval=1) scheduler.run()