def worker(cli_config, burst, logging_level, name, results_ttl, worker_ttl,
job_monitoring_interval, verbose, quiet, sentry_dsn,
exception_handler, pid, queues, rate_limit, **options):
"""Starts an RQ worker."""
settings = read_config_file(cli_config.config) if cli_config.config else {}
# Worker specific default arguments
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet)
try:
cleanup_ghosts(cli_config.connection)
exception_handlers = []
for h in exception_handler:
exception_handlers.append(import_attribute(h))
if is_suspended(cli_config.connection):
click.secho(
'RQ is currently suspended, to resume job execution run "rq resume"',
fg='red')
sys.exit(1)
queues = [
cli_config.queue_class(queue,
connection=cli_config.connection,
job_class=cli_config.job_class)
for queue in queues
]
worker = cli_config.worker_class(
queues,
name=name,
connection=cli_config.connection,
default_worker_ttl=worker_ttl,
default_result_ttl=results_ttl,
job_monitoring_interval=job_monitoring_interval,
job_class=cli_config.job_class,
queue_class=cli_config.queue_class,
exception_handlers=exception_handlers or None,
limit=rate_limit)
# Should we configure Sentry?
if sentry_dsn:
from raven import Client
from raven.transport.http import HTTPTransport
from rq.contrib.sentry import register_sentry
client = Client(sentry_dsn, transport=HTTPTransport)
register_sentry(client, worker)
worker.work(burst=burst, logging_level=logging_level)
except ConnectionError as e:
print(e)
sys.exit(1)
def worker(cli_config, burst, logging_level, name, results_ttl,
worker_ttl, job_monitoring_interval, disable_job_desc_logging, verbose, quiet, sentry_dsn,
exception_handler, pid, disable_default_exception_handler, max_jobs, with_scheduler,
queues, log_format, date_format, **options):
"""Starts an RQ worker."""
settings = read_config_file(cli_config.config) if cli_config.config else {}
# Worker specific default arguments
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
name = name or settings.get('NAME')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet, date_format, log_format)
try:
cleanup_ghosts(cli_config.connection)
exception_handlers = []
for h in exception_handler:
exception_handlers.append(import_attribute(h))
if is_suspended(cli_config.connection):
click.secho('RQ is currently suspended, to resume job execution run "rq resume"', fg='red')
sys.exit(1)
queues = [cli_config.queue_class(queue,
connection=cli_config.connection,
job_class=cli_config.job_class)
for queue in queues]
worker = cli_config.worker_class(
queues, name=name, connection=cli_config.connection,
default_worker_ttl=worker_ttl, default_result_ttl=results_ttl,
job_monitoring_interval=job_monitoring_interval,
job_class=cli_config.job_class, queue_class=cli_config.queue_class,
exception_handlers=exception_handlers or None,
disable_default_exception_handler=disable_default_exception_handler,
log_job_description=not disable_job_desc_logging
)
# Should we configure Sentry?
if sentry_dsn:
from rq.contrib.sentry import register_sentry
register_sentry(sentry_dsn)
# if --verbose or --quiet, override --logging_level
if verbose or quiet:
logging_level = None
worker.work(burst=burst, logging_level=logging_level,
date_format=date_format, log_format=log_format,
max_jobs=max_jobs, with_scheduler=with_scheduler)
except ConnectionError as e:
print(e)
sys.exit(1)
def worker(cli_config, burst, name, results_ttl,
worker_ttl, verbose, quiet, sentry_dsn, exception_handler,
pid, queues, **options):
"""Starts an RQ worker."""
settings = read_config_file(cli_config.config) if cli_config.config else {}
# Worker specific default arguments
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet)
try:
cleanup_ghosts(cli_config.connection)
exception_handlers = []
for h in exception_handler:
exception_handlers.append(import_attribute(h))
if is_suspended(cli_config.connection):
click.secho('RQ is currently suspended, to resume job execution run "rq resume"', fg='red')
sys.exit(1)
queues = [cli_config.queue_class(queue,
connection=cli_config.connection,
job_class=cli_config.job_class)
for queue in queues]
worker = cli_config.worker_class(queues,
name=name,
connection=cli_config.connection,
default_worker_ttl=worker_ttl,
default_result_ttl=results_ttl,
job_class=cli_config.job_class,
queue_class=cli_config.queue_class,
exception_handlers=exception_handlers or None)
# Should we configure Sentry?
if sentry_dsn:
from raven import Client
from raven.transport.http import HTTPTransport
from rq.contrib.sentry import register_sentry
client = Client(sentry_dsn, transport=HTTPTransport)
register_sentry(client, worker)
worker.work(burst=burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def test_config_file(self):
settings = read_config_file("tests.dummy_settings")
self.assertIn("REDIS_HOST", settings)
self.assertEqual(settings['REDIS_HOST'], "testhost.example.com")
def test_config_file(self):
settings = read_config_file("tests.dummy_settings")
self.assertIn("REDIS_HOST", settings)
self.assertEqual(settings['REDIS_HOST'], "testhost.example.com")
def test_config_file(self):
settings = read_config_file('tests.config_files.dummy')
self.assertIn('REDIS_HOST', settings)
self.assertEqual(settings['REDIS_HOST'], 'testhost.example.com')
def test_config_file(self):
settings = read_config_file('tests.config_files.dummy')
self.assertIn('REDIS_HOST', settings)
self.assertEqual(settings['REDIS_HOST'], 'testhost.example.com')
def test_reading_dsn_from_file(self):
settings = read_config_file('tests.config_files.sentry')
self.assertIn('SENTRY_DSN', settings)
self.assertEqual(settings['SENTRY_DSN'], 'https://[email protected]/123')
def test_reading_dsn_from_file(self):
settings = read_config_file('tests.config_files.sentry')
self.assertIn('SENTRY_DSN', settings)
self.assertEqual(settings['SENTRY_DSN'], 'https://[email protected]/123')
def main():
# set up minimal argparser to get -c option
parser = argparse.ArgumentParser(
add_help=False # help will be picked up later when we redfine parser
)
parser.add_argument('-c', "--config", help='Use an rq config file')
args, remaining_argv = parser.parse_known_args()
# config, pass 1: read environment vars
config = {
KEY_HOST : os.environ.get('RQ_REDIS_HOST', 'localhost'),
KEY_PORT : int(os.environ.get('RQ_REDIS_PORT', 6379)),
KEY_DB : int(os.environ.get('RQ_REDIS_DB', 0)),
KEY_PASSWORD : os.environ.get('RQ_REDIS_PASSWORD'),
KEY_URL : os.environ.get('RQ_REDIS_URL')
}
# config, pass 2: read config file
if args.config:
# bit of a hack, this, but does allow helpers.read_config_file to work...
sys.path.insert( 0, os.path.dirname(os.path.realpath(args.config)) )
rq_config = helpers.read_config_file( args.config )
# map rq settings to our own config dict
config[KEY_URL] = rq_config.get("REDIS_URL", config[KEY_URL])
config[KEY_HOST] = rq_config.get("REDIS_HOST", config[KEY_HOST])
config[KEY_PORT] = rq_config.get("REDIS_PORT", config[KEY_PORT])
config[KEY_DB] = rq_config.get("REDIS_DB", config[KEY_DB])
config[KEY_PASSWORD] = rq_config.get("REDIS_PASSWORD",config[KEY_PASSWORD])
# config, pass 3: read commandline args. overwrites any other config.
parser = argparse.ArgumentParser(
parents=[parser] # inherit from existing parser
)
parser.add_argument('-H', '--host', default=config[KEY_HOST], help="Redis host")
parser.add_argument('-p', '--port', default=config[KEY_PORT], type=int, help="Redis port number")
parser.add_argument('-d', '--db', default=config[KEY_DB], type=int, help="Redis database")
parser.add_argument('-P', '--password', default=config[KEY_PASSWORD], help="Redis password")
parser.add_argument('--verbose', '-v', action='store_true', default=False, help='Show more output')
parser.add_argument('--url', '-u', default=config[KEY_URL]
, help='URL describing Redis connection details. \
Overrides other connection arguments if supplied.')
parser.add_argument('-i', '--interval', default=60.0, type=float
, help="How often the scheduler checks for new jobs to add to the \
queue (in seconds, can be floating-point for more precision).")
parser.add_argument('--path', default='.', help='Specify the import path.')
parser.add_argument('--pid', help='A filename to use for the PID file.', metavar='FILE')
args = parser.parse_args( remaining_argv )
if args.path:
sys.path = args.path.split(':') + sys.path
if args.pid:
pid = str(os.getpid())
filename = args.pid
with open(filename, 'w') as f:
f.write(pid)
if args.url is not None:
connection = Redis.from_url(args.url)
else:
connection = Redis(args.host, args.port, args.db, args.password)
if args.verbose:
level = 'DEBUG'
else:
level = 'INFO'
setup_loghandlers(level)
scheduler = Scheduler(connection=connection, interval=args.interval)
scheduler.run()
