def schedule():
"""Creates scheduler object."""
build_scheduler = Scheduler(connection=WorkerQueues.connection)
log.info('scheduler created')
cleanup_interval = int(os.getenv('RENKU_SVC_CLEANUP_INTERVAL', 60))
log.info('cleanup interval set to {}'.format(cleanup_interval))
build_scheduler.schedule(
scheduled_time=datetime.utcnow(),
queue_name=CLEANUP_QUEUE_FILES,
func=cache_files_cleanup,
interval=cleanup_interval,
result_ttl=cleanup_interval + 1,
)
build_scheduler.schedule(
scheduled_time=datetime.utcnow(),
queue_name=CLEANUP_QUEUE_PROJECTS,
func=cache_project_cleanup,
interval=cleanup_interval,
result_ttl=cleanup_interval + 1,
)
log_level = os.getenv('RQ_WORKER_LOG_LEVEL', 'INFO')
setup_loghandlers(log_level)
log.info('log level set to {}'.format(log_level))
yield build_scheduler
def run(self, verbose, interval, pid):
scheduler = self.rq.get_scheduler(interval)
if pid:
with open(os.path.expanduser(pid), 'w') as fp:
fp.write(str(os.getpid()))
if verbose:
level = 'DEBUG'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler.run()
def run(self, verbose, interval, pid):
scheduler = self.rq.get_scheduler(interval)
if pid:
with open(os.path.expanduser(pid), 'w') as fp:
fp.write(str(os.getpid()))
if verbose:
level = 'DEBUG'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler.run()
def main():
parser = argparse.ArgumentParser(description="Runs RQ scheduler")
parser.add_argument(
"-b", "--burst", action="store_true", default=False, help="Run in burst mode (quit after all work is done)"
)
parser.add_argument("-H", "--host", default=os.environ.get("RQ_REDIS_HOST", "localhost"), help="Redis host")
parser.add_argument(
"-p", "--port", default=int(os.environ.get("RQ_REDIS_PORT", 6379)), type=int, help="Redis port number"
)
parser.add_argument("-d", "--db", default=int(os.environ.get("RQ_REDIS_DB", 0)), type=int, help="Redis database")
parser.add_argument("-P", "--password", default=os.environ.get("RQ_REDIS_PASSWORD"), help="Redis password")
parser.add_argument("--verbose", "-v", action="store_true", default=False, help="Show more output")
parser.add_argument(
"--url",
"-u",
default=os.environ.get("RQ_REDIS_URL"),
help="URL describing Redis connection details. \
Overrides other connection arguments if supplied.",
)
parser.add_argument(
"-i",
"--interval",
default=60.0,
type=float,
help="How often the scheduler checks for new jobs to add to the \
queue (in seconds, can be floating-point for more precision).",
)
parser.add_argument("--path", default=".", help="Specify the import path.")
parser.add_argument("--pid", help="A filename to use for the PID file.", metavar="FILE")
args = parser.parse_args()
if args.path:
sys.path = args.path.split(":") + sys.path
if args.pid:
pid = str(os.getpid())
filename = args.pid
with open(filename, "w") as f:
f.write(pid)
if args.url is not None:
connection = Redis.from_url(args.url)
else:
connection = Redis(args.host, args.port, args.db, args.password)
if args.verbose:
level = "DEBUG"
else:
level = "INFO"
setup_loghandlers(level)
scheduler = Scheduler(connection=connection, interval=args.interval)
scheduler.run(burst=args.burst)
def scheduler(rq, ctx, verbose, interval, pid):
"Periodically checks for scheduled jobs."
scheduler = rq.get_scheduler(interval)
if pid:
with open(os.path.expanduser(pid), 'w') as fp:
fp.write(str(os.getpid()))
if verbose:
level = 'DEBUG'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler.run()
def create_scheduler(queue: Queue,
config: Dict[str, Any],
job_class: Job = Job) -> ChaosPlatformScheduler:
"""
Create a scheduler on the given queue and job class
"""
log_level = 'DEBUG' if config.get('debug') else 'INFO'
setup_loghandlers(level=log_level)
return ChaosPlatformScheduler(queue=queue,
connection=queue.connection,
queue_class=queue.__class__,
job_class=job_class)
def scheduler(rq, ctx, verbose, burst, queue, interval, pid):
"Periodically checks for scheduled jobs."
scheduler = rq.get_scheduler(interval=interval, queue=queue)
if pid:
with open(os.path.expanduser(pid), 'w') as fp:
fp.write(str(os.getpid()))
if verbose:
level = 'DEBUG'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler.run(burst=burst)
def main():
parser = argparse.ArgumentParser(description='Runs RQ scheduler')
parser.add_argument('-b', '--burst', action='store_true', default=False, help='Run in burst mode (quit after all work is done)')
parser.add_argument('-H', '--host', default=os.environ.get('RQ_REDIS_HOST', 'localhost'), help="Redis host")
parser.add_argument('-p', '--port', default=int(os.environ.get('RQ_REDIS_PORT', 6379)), type=int, help="Redis port number")
parser.add_argument('-d', '--db', default=int(os.environ.get('RQ_REDIS_DB', 0)), type=int, help="Redis database")
parser.add_argument('-P', '--password', default=os.environ.get('RQ_REDIS_PASSWORD'), help="Redis password")
parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Show more output')
parser.add_argument('--quiet', action='store_true', default=False, help='Show less output')
parser.add_argument('--url', '-u', default=os.environ.get('RQ_REDIS_URL')
, help='URL describing Redis connection details. \
Overrides other connection arguments if supplied.')
parser.add_argument('-i', '--interval', default=60.0, type=float
, help="How often the scheduler checks for new jobs to add to the \
queue (in seconds, can be floating-point for more precision).")
parser.add_argument('--path', default='.', help='Specify the import path.')
parser.add_argument('--pid', help='A filename to use for the PID file.', metavar='FILE')
parser.add_argument('-j', '--job-class', help='Custom RQ Job class')
parser.add_argument('-q', '--queue-class', help='Custom RQ Queue class')
args = parser.parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
if args.pid:
pid = str(os.getpid())
filename = args.pid
with open(filename, 'w') as f:
f.write(pid)
if args.url is not None:
connection = Redis.from_url(args.url)
else:
connection = Redis(args.host, args.port, args.db, args.password)
if args.verbose:
level = 'DEBUG'
elif args.quiet:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler = Scheduler(connection=connection,
interval=args.interval,
job_class=args.job_class,
queue_class=args.queue_class)
scheduler.run(burst=args.burst)
def main():
redis_connection = StrictRedis(host=ENV_DICT.get('REDIS_HOST',
'localhost'),
port=ENV_DICT.get('REDIS_PORT', 6379),
db=ENV_DICT.get('REDIS_DB', 0),
password=ENV_DICT.get('REDIS_PASSWORD'))
interval = ENV_DICT.get('RQ_SCHEDULER_POLLING_INTERVAL', 60)
verbose = ENV_DICT.get('RQ_SCHEDULER_VERBOSE_OUTPUT', False)
burst = ENV_DICT.get('RQ_SCHEDULER_BURST_MODE', False)
if verbose:
log_level = 'DEBUG'
else:
log_level = 'INFO'
setup_loghandlers(log_level)
scheduler = Scheduler(connection=redis_connection, interval=interval)
scheduler.run(burst=burst)
def handle(self, *args, **options):
pid = options.get('pid')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
# Verbosity is defined by default in BaseCommand for all commands
verbosity = options.get('verbosity')
if verbosity >= 2:
level = 'DEBUG'
elif verbosity == 0:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler = get_scheduler(name=options.get('queue'),
interval=options.get('interval'))
scheduler.run()
def handle(self, *args, **options):
pid = options.get('pid')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
# Verbosity is defined by default in BaseCommand for all commands
verbosity = options.get('verbosity')
if verbosity >= 2:
level = 'DEBUG'
elif verbosity == 0:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler = get_scheduler(
name=options.get('queue'), interval=options.get('interval'))
scheduler.run()
def schedule():
"""Creates scheduler object."""
setup_loghandlers(level=DEPLOYMENT_LOG_LEVEL)
build_scheduler = Scheduler(connection=WorkerQueues.connection)
scheduler_log.info("scheduler created")
cleanup_interval = int(os.getenv("RENKU_SVC_CLEANUP_INTERVAL", 60))
scheduler_log.info("cleanup interval set to {}".format(cleanup_interval))
def requeue(*args, **kwargs):
"""Inverval check for scheduled jobs."""
job = args[0]
queue = Scheduler.get_queue_for_job(build_scheduler, job)
scheduler_log.info(
f"job {job.id}:{job.func_name} re/queued to {queue.name}")
return queue
# NOTE: Patch scheduler to have requeing information on INFO log level.
build_scheduler.get_queue_for_job = requeue
build_scheduler.schedule(
scheduled_time=datetime.utcnow(),
queue_name=CLEANUP_QUEUE_FILES,
func=cache_files_cleanup,
interval=cleanup_interval,
result_ttl=cleanup_interval + 1,
)
build_scheduler.schedule(
scheduled_time=datetime.utcnow(),
queue_name=CLEANUP_QUEUE_PROJECTS,
func=cache_project_cleanup,
interval=cleanup_interval,
result_ttl=cleanup_interval + 1,
)
scheduler_log.info(f"log level set to {DEPLOYMENT_LOG_LEVEL}")
yield build_scheduler
def run_scheduler():
"""Initializes a rq scheduler."""
conn = Redis(
host=app.config['RQ_DEFAULT_HOST'],
port=app.config['RQ_DEFAULT_PORT'],
db=0,
password=app.config['RQ_DEFAULT_PASSWORD']
)
setup_loghandlers('INFO')
scheduler = Scheduler(connection=conn, interval=60.0)
for _ in xrange(10):
try:
scheduler.run()
except ValueError as exc:
if exc.message == 'There\'s already an active RQ scheduler':
scheduler.log.info(
'An RQ scheduler instance is already running. Retrying in '
'%d seconds.', 10,
)
time.sleep(10)
else:
raise exc
def run_scheduler():
"""Initializes a rq scheduler."""
conn = Redis(
host=app.config['RQ_DEFAULT_HOST'],
port=app.config['RQ_DEFAULT_PORT'],
db=0,
password=app.config['RQ_DEFAULT_PASSWORD']
)
setup_loghandlers('INFO')
scheduler = Scheduler(connection=conn, interval=60.0)
for _ in range(10):
try:
scheduler.run()
except ValueError as exc:
if exc.message == 'There\'s already an active RQ scheduler':
scheduler.log.info(
'An RQ scheduler instance is already running. Retrying in '
'%d seconds.', 10,
)
time.sleep(10)
else:
raise exc
def main():
parser = argparse.ArgumentParser(description='Runs RQ scheduler')
parser.add_argument('-b',
'--burst',
action='store_true',
default=False,
help='Run in burst mode (quit after all work is done)')
parser.add_argument('-H',
'--host',
default=os.environ.get('RQ_REDIS_HOST', 'localhost'),
help="Redis host")
parser.add_argument('-p',
'--port',
default=int(os.environ.get('RQ_REDIS_PORT', 6379)),
type=int,
help="Redis port number")
parser.add_argument('-d',
'--db',
default=int(os.environ.get('RQ_REDIS_DB', 0)),
type=int,
help="Redis database")
parser.add_argument('-P',
'--password',
default=os.environ.get('RQ_REDIS_PASSWORD'),
help="Redis password")
parser.add_argument('--verbose',
'-v',
action='store_true',
default=False,
help='Show more output')
parser.add_argument('--quiet',
'-q',
action='store_true',
default=False,
help='Show less output')
parser.add_argument('--url',
'-u',
default=os.environ.get('RQ_REDIS_URL'),
help='URL describing Redis connection details. \
Overrides other connection arguments if supplied.')
parser.add_argument(
'-i',
'--interval',
default=60.0,
type=float,
help="How often the scheduler checks for new jobs to add to the \
queue (in seconds, can be floating-point for more precision).")
parser.add_argument('--path', default='.', help='Specify the import path.')
parser.add_argument('--pid',
help='A filename to use for the PID file.',
metavar='FILE')
parser.add_argument('-j', '--job-class', help='Custom RQ Job class')
parser.add_argument('-q', '--queue-class', help='Custom RQ Queue class')
args = parser.parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
if args.pid:
pid = str(os.getpid())
filename = args.pid
with open(filename, 'w') as f:
f.write(pid)
if args.url is not None:
connection = Redis.from_url(args.url)
else:
connection = Redis(args.host, args.port, args.db, args.password)
if args.verbose:
level = 'DEBUG'
elif args.quiet:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler = Scheduler(connection=connection,
interval=args.interval,
job_class=args.job_class,
queue_class=args.queue_class)
scheduler.run(burst=args.burst)
def run_scheduler():
"""rq-scheduler for periodic tasks"""
if app.debug:
setup_loghandlers('DEBUG')
scheduler = Scheduler(connection=get_connection(), interval=30)
scheduler.run()
