def schedule(): """Creates scheduler object.""" build_scheduler = Scheduler(connection=WorkerQueues.connection) log.info('scheduler created') cleanup_interval = int(os.getenv('RENKU_SVC_CLEANUP_INTERVAL', 60)) log.info('cleanup interval set to {}'.format(cleanup_interval)) build_scheduler.schedule( scheduled_time=datetime.utcnow(), queue_name=CLEANUP_QUEUE_FILES, func=cache_files_cleanup, interval=cleanup_interval, result_ttl=cleanup_interval + 1, ) build_scheduler.schedule( scheduled_time=datetime.utcnow(), queue_name=CLEANUP_QUEUE_PROJECTS, func=cache_project_cleanup, interval=cleanup_interval, result_ttl=cleanup_interval + 1, ) log_level = os.getenv('RQ_WORKER_LOG_LEVEL', 'INFO') setup_loghandlers(log_level) log.info('log level set to {}'.format(log_level)) yield build_scheduler
def run(self, verbose, interval, pid): scheduler = self.rq.get_scheduler(interval) if pid: with open(os.path.expanduser(pid), 'w') as fp: fp.write(str(os.getpid())) if verbose: level = 'DEBUG' else: level = 'INFO' setup_loghandlers(level) scheduler.run()
def main(): parser = argparse.ArgumentParser(description="Runs RQ scheduler") parser.add_argument( "-b", "--burst", action="store_true", default=False, help="Run in burst mode (quit after all work is done)" ) parser.add_argument("-H", "--host", default=os.environ.get("RQ_REDIS_HOST", "localhost"), help="Redis host") parser.add_argument( "-p", "--port", default=int(os.environ.get("RQ_REDIS_PORT", 6379)), type=int, help="Redis port number" ) parser.add_argument("-d", "--db", default=int(os.environ.get("RQ_REDIS_DB", 0)), type=int, help="Redis database") parser.add_argument("-P", "--password", default=os.environ.get("RQ_REDIS_PASSWORD"), help="Redis password") parser.add_argument("--verbose", "-v", action="store_true", default=False, help="Show more output") parser.add_argument( "--url", "-u", default=os.environ.get("RQ_REDIS_URL"), help="URL describing Redis connection details. \ Overrides other connection arguments if supplied.", ) parser.add_argument( "-i", "--interval", default=60.0, type=float, help="How often the scheduler checks for new jobs to add to the \ queue (in seconds, can be floating-point for more precision).", ) parser.add_argument("--path", default=".", help="Specify the import path.") parser.add_argument("--pid", help="A filename to use for the PID file.", metavar="FILE") args = parser.parse_args() if args.path: sys.path = args.path.split(":") + sys.path if args.pid: pid = str(os.getpid()) filename = args.pid with open(filename, "w") as f: f.write(pid) if args.url is not None: connection = Redis.from_url(args.url) else: connection = Redis(args.host, args.port, args.db, args.password) if args.verbose: level = "DEBUG" else: level = "INFO" setup_loghandlers(level) scheduler = Scheduler(connection=connection, interval=args.interval) scheduler.run(burst=args.burst)
def scheduler(rq, ctx, verbose, interval, pid): "Periodically checks for scheduled jobs." scheduler = rq.get_scheduler(interval) if pid: with open(os.path.expanduser(pid), 'w') as fp: fp.write(str(os.getpid())) if verbose: level = 'DEBUG' else: level = 'INFO' setup_loghandlers(level) scheduler.run()
def create_scheduler(queue: Queue, config: Dict[str, Any], job_class: Job = Job) -> ChaosPlatformScheduler: """ Create a scheduler on the given queue and job class """ log_level = 'DEBUG' if config.get('debug') else 'INFO' setup_loghandlers(level=log_level) return ChaosPlatformScheduler(queue=queue, connection=queue.connection, queue_class=queue.__class__, job_class=job_class)
def scheduler(rq, ctx, verbose, burst, queue, interval, pid): "Periodically checks for scheduled jobs." scheduler = rq.get_scheduler(interval=interval, queue=queue) if pid: with open(os.path.expanduser(pid), 'w') as fp: fp.write(str(os.getpid())) if verbose: level = 'DEBUG' else: level = 'INFO' setup_loghandlers(level) scheduler.run(burst=burst)
def main(): parser = argparse.ArgumentParser(description='Runs RQ scheduler') parser.add_argument('-b', '--burst', action='store_true', default=False, help='Run in burst mode (quit after all work is done)') parser.add_argument('-H', '--host', default=os.environ.get('RQ_REDIS_HOST', 'localhost'), help="Redis host") parser.add_argument('-p', '--port', default=int(os.environ.get('RQ_REDIS_PORT', 6379)), type=int, help="Redis port number") parser.add_argument('-d', '--db', default=int(os.environ.get('RQ_REDIS_DB', 0)), type=int, help="Redis database") parser.add_argument('-P', '--password', default=os.environ.get('RQ_REDIS_PASSWORD'), help="Redis password") parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Show more output') parser.add_argument('--quiet', action='store_true', default=False, help='Show less output') parser.add_argument('--url', '-u', default=os.environ.get('RQ_REDIS_URL') , help='URL describing Redis connection details. \ Overrides other connection arguments if supplied.') parser.add_argument('-i', '--interval', default=60.0, type=float , help="How often the scheduler checks for new jobs to add to the \ queue (in seconds, can be floating-point for more precision).") parser.add_argument('--path', default='.', help='Specify the import path.') parser.add_argument('--pid', help='A filename to use for the PID file.', metavar='FILE') parser.add_argument('-j', '--job-class', help='Custom RQ Job class') parser.add_argument('-q', '--queue-class', help='Custom RQ Queue class') args = parser.parse_args() if args.path: sys.path = args.path.split(':') + sys.path if args.pid: pid = str(os.getpid()) filename = args.pid with open(filename, 'w') as f: f.write(pid) if args.url is not None: connection = Redis.from_url(args.url) else: connection = Redis(args.host, args.port, args.db, args.password) if args.verbose: level = 'DEBUG' elif args.quiet: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) scheduler = Scheduler(connection=connection, interval=args.interval, job_class=args.job_class, queue_class=args.queue_class) scheduler.run(burst=args.burst)
def main(): redis_connection = StrictRedis(host=ENV_DICT.get('REDIS_HOST', 'localhost'), port=ENV_DICT.get('REDIS_PORT', 6379), db=ENV_DICT.get('REDIS_DB', 0), password=ENV_DICT.get('REDIS_PASSWORD')) interval = ENV_DICT.get('RQ_SCHEDULER_POLLING_INTERVAL', 60) verbose = ENV_DICT.get('RQ_SCHEDULER_VERBOSE_OUTPUT', False) burst = ENV_DICT.get('RQ_SCHEDULER_BURST_MODE', False) if verbose: log_level = 'DEBUG' else: log_level = 'INFO' setup_loghandlers(log_level) scheduler = Scheduler(connection=redis_connection, interval=interval) scheduler.run(burst=burst)
def handle(self, *args, **options): pid = options.get('pid') if pid: with open(os.path.expanduser(pid), "w") as fp: fp.write(str(os.getpid())) # Verbosity is defined by default in BaseCommand for all commands verbosity = options.get('verbosity') if verbosity >= 2: level = 'DEBUG' elif verbosity == 0: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) scheduler = get_scheduler(name=options.get('queue'), interval=options.get('interval')) scheduler.run()
def handle(self, *args, **options): pid = options.get('pid') if pid: with open(os.path.expanduser(pid), "w") as fp: fp.write(str(os.getpid())) # Verbosity is defined by default in BaseCommand for all commands verbosity = options.get('verbosity') if verbosity >= 2: level = 'DEBUG' elif verbosity == 0: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) scheduler = get_scheduler( name=options.get('queue'), interval=options.get('interval')) scheduler.run()
def schedule(): """Creates scheduler object.""" setup_loghandlers(level=DEPLOYMENT_LOG_LEVEL) build_scheduler = Scheduler(connection=WorkerQueues.connection) scheduler_log.info("scheduler created") cleanup_interval = int(os.getenv("RENKU_SVC_CLEANUP_INTERVAL", 60)) scheduler_log.info("cleanup interval set to {}".format(cleanup_interval)) def requeue(*args, **kwargs): """Inverval check for scheduled jobs.""" job = args[0] queue = Scheduler.get_queue_for_job(build_scheduler, job) scheduler_log.info( f"job {job.id}:{job.func_name} re/queued to {queue.name}") return queue # NOTE: Patch scheduler to have requeing information on INFO log level. build_scheduler.get_queue_for_job = requeue build_scheduler.schedule( scheduled_time=datetime.utcnow(), queue_name=CLEANUP_QUEUE_FILES, func=cache_files_cleanup, interval=cleanup_interval, result_ttl=cleanup_interval + 1, ) build_scheduler.schedule( scheduled_time=datetime.utcnow(), queue_name=CLEANUP_QUEUE_PROJECTS, func=cache_project_cleanup, interval=cleanup_interval, result_ttl=cleanup_interval + 1, ) scheduler_log.info(f"log level set to {DEPLOYMENT_LOG_LEVEL}") yield build_scheduler
def run_scheduler(): """Initializes a rq scheduler.""" conn = Redis( host=app.config['RQ_DEFAULT_HOST'], port=app.config['RQ_DEFAULT_PORT'], db=0, password=app.config['RQ_DEFAULT_PASSWORD'] ) setup_loghandlers('INFO') scheduler = Scheduler(connection=conn, interval=60.0) for _ in xrange(10): try: scheduler.run() except ValueError as exc: if exc.message == 'There\'s already an active RQ scheduler': scheduler.log.info( 'An RQ scheduler instance is already running. Retrying in ' '%d seconds.', 10, ) time.sleep(10) else: raise exc
def run_scheduler(): """Initializes a rq scheduler.""" conn = Redis( host=app.config['RQ_DEFAULT_HOST'], port=app.config['RQ_DEFAULT_PORT'], db=0, password=app.config['RQ_DEFAULT_PASSWORD'] ) setup_loghandlers('INFO') scheduler = Scheduler(connection=conn, interval=60.0) for _ in range(10): try: scheduler.run() except ValueError as exc: if exc.message == 'There\'s already an active RQ scheduler': scheduler.log.info( 'An RQ scheduler instance is already running. Retrying in ' '%d seconds.', 10, ) time.sleep(10) else: raise exc
def main(): parser = argparse.ArgumentParser(description='Runs RQ scheduler') parser.add_argument('-b', '--burst', action='store_true', default=False, help='Run in burst mode (quit after all work is done)') parser.add_argument('-H', '--host', default=os.environ.get('RQ_REDIS_HOST', 'localhost'), help="Redis host") parser.add_argument('-p', '--port', default=int(os.environ.get('RQ_REDIS_PORT', 6379)), type=int, help="Redis port number") parser.add_argument('-d', '--db', default=int(os.environ.get('RQ_REDIS_DB', 0)), type=int, help="Redis database") parser.add_argument('-P', '--password', default=os.environ.get('RQ_REDIS_PASSWORD'), help="Redis password") parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Show more output') parser.add_argument('--quiet', '-q', action='store_true', default=False, help='Show less output') parser.add_argument('--url', '-u', default=os.environ.get('RQ_REDIS_URL'), help='URL describing Redis connection details. \ Overrides other connection arguments if supplied.') parser.add_argument( '-i', '--interval', default=60.0, type=float, help="How often the scheduler checks for new jobs to add to the \ queue (in seconds, can be floating-point for more precision).") parser.add_argument('--path', default='.', help='Specify the import path.') parser.add_argument('--pid', help='A filename to use for the PID file.', metavar='FILE') parser.add_argument('-j', '--job-class', help='Custom RQ Job class') parser.add_argument('-q', '--queue-class', help='Custom RQ Queue class') args = parser.parse_args() if args.path: sys.path = args.path.split(':') + sys.path if args.pid: pid = str(os.getpid()) filename = args.pid with open(filename, 'w') as f: f.write(pid) if args.url is not None: connection = Redis.from_url(args.url) else: connection = Redis(args.host, args.port, args.db, args.password) if args.verbose: level = 'DEBUG' elif args.quiet: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) scheduler = Scheduler(connection=connection, interval=args.interval, job_class=args.job_class, queue_class=args.queue_class) scheduler.run(burst=args.burst)
def run_scheduler(): """rq-scheduler for periodic tasks""" if app.debug: setup_loghandlers('DEBUG') scheduler = Scheduler(connection=get_connection(), interval=30) scheduler.run()