def start_worker(self, node=HOSTNAME, logging_level="INFO"):
if manager.debug:
logger.warn("running in debug mode")
return
name = "{0}:{1}".format(node, uuid.uuid4().hex[:8])
q_names = self.queues
cleanup_ghosts(manager.connection)
rqs = [
Queue(queue, connection=manager.connection) for queue in q_names
]
worker = FlaskWorker(
rqs,
name=name,
connection=manager.connection,
default_worker_ttl=manager.worker_ttl,
default_result_ttl=manager.result_ttl,
job_class=Job,
queue_class=Queue,
exception_handlers=None,
)
if manager.sentry_dsn:
client = Client(manager.sentry_dsn)
register_sentry(client, worker)
worker.work(burst=False, logging_level=logging_level)
def auto_worker(queue_name, redis_url, skip_failed, default_result_ttl):
"""
Internal target to use in multiprocessing
"""
connection = Redis.from_url(redis_url)
cleanup_ghosts(connection)
worker_class = import_attribute("rq.Worker")
queue_class = import_attribute("rq.Queue")
queue = queue_class(queue_name, connection=connection)
if skip_failed:
exception_handlers = []
else:
exception_handlers = None
name = "{}-auto".format(uuid4().hex)
worker = worker_class(
queue,
name=name,
connection=connection,
exception_handlers=exception_handlers,
default_result_ttl=default_result_ttl,
)
worker.work(burst=True)
def worker(cli_config, burst, logging_level, name, results_ttl, worker_ttl,
job_monitoring_interval, verbose, quiet, sentry_dsn,
exception_handler, pid, queues, rate_limit, **options):
"""Starts an RQ worker."""
settings = read_config_file(cli_config.config) if cli_config.config else {}
# Worker specific default arguments
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet)
try:
cleanup_ghosts(cli_config.connection)
exception_handlers = []
for h in exception_handler:
exception_handlers.append(import_attribute(h))
if is_suspended(cli_config.connection):
click.secho(
'RQ is currently suspended, to resume job execution run "rq resume"',
fg='red')
sys.exit(1)
queues = [
cli_config.queue_class(queue,
connection=cli_config.connection,
job_class=cli_config.job_class)
for queue in queues
]
worker = cli_config.worker_class(
queues,
name=name,
connection=cli_config.connection,
default_worker_ttl=worker_ttl,
default_result_ttl=results_ttl,
job_monitoring_interval=job_monitoring_interval,
job_class=cli_config.job_class,
queue_class=cli_config.queue_class,
exception_handlers=exception_handlers or None,
limit=rate_limit)
# Should we configure Sentry?
if sentry_dsn:
from raven import Client
from raven.transport.http import HTTPTransport
from rq.contrib.sentry import register_sentry
client = Client(sentry_dsn, transport=HTTPTransport)
register_sentry(client, worker)
worker.work(burst=burst, logging_level=logging_level)
except ConnectionError as e:
print(e)
sys.exit(1)
def worker(url, config, burst, name, worker_class, job_class, queue_class,
connection_class, path, results_ttl, worker_ttl, verbose, quiet,
sentry_dsn, exception_handler, pid, queues):
"""Starts an RQ worker."""
if path:
sys.path = path.split(':') + sys.path
settings = read_config_file(config) if config else {}
# Worker specific default arguments
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet)
connection_class = import_attribute(connection_class)
conn = connect(url, config, connection_class)
cleanup_ghosts(conn)
worker_class = import_attribute(worker_class)
queue_class = import_attribute(queue_class)
exception_handlers = []
for h in exception_handler:
exception_handlers.append(import_attribute(h))
if is_suspended(conn):
click.secho(
'RQ is currently suspended, to resume job execution run "rq resume"',
fg='red')
sys.exit(1)
try:
queues = [queue_class(queue, connection=conn) for queue in queues]
w = worker_class(queues,
name=name,
connection=conn,
default_worker_ttl=worker_ttl,
default_result_ttl=results_ttl,
job_class=job_class,
queue_class=queue_class,
exception_handlers=exception_handlers or None)
# Should we configure Sentry?
if sentry_dsn:
from raven import Client
from raven.transport.http import HTTPTransport
from rq.contrib.sentry import register_sentry
client = Client(sentry_dsn, transport=HTTPTransport)
register_sentry(client, w)
w.work(burst=burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def worker(cli_config, burst, logging_level, name, results_ttl,
worker_ttl, job_monitoring_interval, disable_job_desc_logging, verbose, quiet, sentry_dsn,
exception_handler, pid, disable_default_exception_handler, max_jobs, with_scheduler,
queues, log_format, date_format, **options):
"""Starts an RQ worker."""
settings = read_config_file(cli_config.config) if cli_config.config else {}
# Worker specific default arguments
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
name = name or settings.get('NAME')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet, date_format, log_format)
try:
cleanup_ghosts(cli_config.connection)
exception_handlers = []
for h in exception_handler:
exception_handlers.append(import_attribute(h))
if is_suspended(cli_config.connection):
click.secho('RQ is currently suspended, to resume job execution run "rq resume"', fg='red')
sys.exit(1)
queues = [cli_config.queue_class(queue,
connection=cli_config.connection,
job_class=cli_config.job_class)
for queue in queues]
worker = cli_config.worker_class(
queues, name=name, connection=cli_config.connection,
default_worker_ttl=worker_ttl, default_result_ttl=results_ttl,
job_monitoring_interval=job_monitoring_interval,
job_class=cli_config.job_class, queue_class=cli_config.queue_class,
exception_handlers=exception_handlers or None,
disable_default_exception_handler=disable_default_exception_handler,
log_job_description=not disable_job_desc_logging
)
# Should we configure Sentry?
if sentry_dsn:
from rq.contrib.sentry import register_sentry
register_sentry(sentry_dsn)
# if --verbose or --quiet, override --logging_level
if verbose or quiet:
logging_level = None
worker.work(burst=burst, logging_level=logging_level,
date_format=date_format, log_format=log_format,
max_jobs=max_jobs, with_scheduler=with_scheduler)
except ConnectionError as e:
print(e)
sys.exit(1)
def worker(url, config, burst, name, worker_class, job_class, queue_class, path, results_ttl, worker_ttl,
verbose, quiet, sentry_dsn, exception_handler, pid, queues):
"""Starts an RQ worker."""
if path:
sys.path = path.split(':') + sys.path
settings = read_config_file(config) if config else {}
# Worker specific default arguments
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet)
conn = connect(url, config)
cleanup_ghosts(conn)
worker_class = import_attribute(worker_class)
queue_class = import_attribute(queue_class)
exception_handlers = []
for h in exception_handler:
exception_handlers.append(import_attribute(h))
if is_suspended(conn):
click.secho('RQ is currently suspended, to resume job execution run "rq resume"', fg='red')
sys.exit(1)
try:
queues = [queue_class(queue, connection=conn) for queue in queues]
w = worker_class(queues,
name=name,
connection=conn,
default_worker_ttl=worker_ttl,
default_result_ttl=results_ttl,
job_class=job_class,
queue_class=queue_class,
exception_handlers=exception_handlers or None)
# Should we configure Sentry?
if sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
from raven.transport.http import HTTPTransport
client = Client(sentry_dsn, transport=HTTPTransport)
register_sentry(client, w)
w.work(burst=burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.pid:
with open(os.path.expanduser(args.pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(args)
setup_redis(args)
cleanup_ghosts()
worker_class = import_attribute(args.worker_class)
queue_class = import_attribute(args.queue_class)
try:
queues = list(map(queue_class, args.queues))
w = worker_class(queues,
name=args.name,
default_worker_ttl=args.worker_ttl,
default_result_ttl=args.results_ttl,
job_class=args.job_class)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def worker(url, config, burst, name, worker_class, job_class, queue_class,
path, results_ttl, worker_ttl, verbose, quiet, sentry_dsn, pid,
queues):
"""Starts an RQ worker."""
if path:
sys.path = path.split(':') + sys.path
settings = read_config_file(config) if config else {}
# Worker specific default arguments
url = url or settings.get('REDIS_URL')
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet)
conn = connect(url)
cleanup_ghosts(conn)
worker_class = import_attribute(worker_class)
queue_class = import_attribute(queue_class)
if worker_class.paused():
click.secho("The worker has been paused, run reset_paused", fg='red')
sys.exit(1)
try:
queues = [queue_class(queue, connection=conn) for queue in queues]
w = worker_class(queues,
name=name,
connection=conn,
default_worker_ttl=worker_ttl,
default_result_ttl=results_ttl,
job_class=job_class)
# Should we configure Sentry?
if sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(sentry_dsn)
register_sentry(client, w)
w.work(burst=burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def worker(cli_config, burst, logging_level, name, results_ttl,
worker_ttl, job_monitoring_interval, verbose, quiet, sentry_dsn,
exception_handler, pid, disable_default_exception_handler, queues,
log_format, date_format, **options):
"""Starts an RQ worker."""
settings = read_config_file(cli_config.config) if cli_config.config else {}
# Worker specific default arguments
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
name = name or settings.get('NAME')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet, date_format, log_format)
try:
cleanup_ghosts(cli_config.connection)
exception_handlers = []
for h in exception_handler:
exception_handlers.append(import_attribute(h))
if is_suspended(cli_config.connection):
click.secho('RQ is currently suspended, to resume job execution run "rq resume"', fg='red')
sys.exit(1)
queues = [cli_config.queue_class(queue,
connection=cli_config.connection,
job_class=cli_config.job_class)
for queue in queues]
worker = cli_config.worker_class(
queues, name=name, connection=cli_config.connection,
default_worker_ttl=worker_ttl, default_result_ttl=results_ttl,
job_monitoring_interval=job_monitoring_interval,
job_class=cli_config.job_class, queue_class=cli_config.queue_class,
exception_handlers=exception_handlers or None,
disable_default_exception_handler=disable_default_exception_handler
)
# Should we configure Sentry?
if sentry_dsn:
from rq.contrib.sentry import register_sentry
register_sentry(sentry_dsn)
worker.work(burst=burst, logging_level=logging_level, date_format=date_format, log_format=log_format)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.pid:
with open(os.path.expanduser(args.pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(args)
setup_redis(args)
cleanup_ghosts()
worker_class = import_attribute(args.worker_class)
try:
queues = list(map(Queue, args.queues))
w = worker_class(queues,
name=args.name,
default_worker_ttl=args.worker_ttl,
default_result_ttl=args.results_ttl,
job_class=args.job_class)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def worker(url, config, burst, name, worker_class, job_class, queue_class, path, results_ttl, worker_ttl,
verbose, quiet, sentry_dsn, pid, queues):
"""Starts an RQ worker."""
if path:
sys.path = path.split(':') + sys.path
settings = read_config_file(config) if config else {}
# Worker specific default arguments
url = url or settings.get('REDIS_URL')
queues = queues or settings.get('QUEUES', ['default'])
sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN')
if pid:
with open(os.path.expanduser(pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(verbose, quiet)
conn = connect(url)
cleanup_ghosts(conn)
worker_class = import_attribute(worker_class)
queue_class = import_attribute(queue_class)
if worker_class.paused():
click.secho("The worker has been paused, run reset_paused", fg='red')
sys.exit(1)
try:
queues = [queue_class(queue, connection=conn) for queue in queues]
w = worker_class(queues,
name=name,
connection=conn,
default_worker_ttl=worker_ttl,
default_result_ttl=results_ttl,
job_class=job_class)
# Should we configure Sentry?
if sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(sentry_dsn)
register_sentry(client, w)
w.work(burst=burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Other default arguments
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.verbose and args.quiet:
raise RuntimeError(
"Flags --verbose and --quiet are mutually exclusive.")
if args.verbose:
level = 'DEBUG'
elif args.quiet:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
setup_redis(args)
cleanup_ghosts()
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.pid:
with open(os.path.expanduser(args.pid), "w") as fp:
fp.write(str(os.getpid())) # 把 pid 写到 home 目录下
setup_loghandlers_from_args(args)
setup_redis(args) # 启动 redis，创建好 connection 压入栈內，下面的 Queue、Worker 后续都会取到
cleanup_ghosts() # 清理 ghost worker
worker_class = import_attribute(args.worker_class)
try:
queues = list(map(Queue, args.queues)) # 实例化 queue，默认是 default queue
w = worker_class(queues, name=args.name) # 实例化 worker
# Should we configure Sentry?
# raven 是 Sentry（用于监控代码发生异常时通知维护人员）的 python 客户端
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst) # 启动 worker
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Other default arguments
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.verbose and args.quiet:
raise RuntimeError("Flags --verbose and --quiet are mutually exclusive.")
if args.verbose:
level = 'DEBUG'
elif args.quiet:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
setup_redis(args)
cleanup_ghosts()
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN', None)
setup_loghandlers_from_args(args)
setup_redis(args)
cleanup_ghosts()
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def run(name: str,
queue: str,
connection_host: str,
connection_port: int = 6379,
connection_password: str = None,
connection_over_tls: bool = False,
connection_db: int = 0,
verbose: bool = False):
date_format = "%Y-%m-%dT%H:%M:%S"
log_format = "%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s"
setup_loghandlers_from_args(verbose, False, date_format, log_format)
params = {
"ssl": connection_over_tls,
"password": connection_password,
"host": connection_host,
"port": connection_port,
"db": connection_db
}
try:
c = Redis(**params)
cleanup_ghosts(c)
worker = Worker([queue],
name=name,
connection=c,
default_worker_ttl=420,
default_result_ttl=500,
job_monitoring_interval=30,
job_class=Job,
queue_class=Queue,
exception_handlers=None)
worker.work(burst=False,
logging_level="DEBUG" if verbose else "INFO",
date_format=date_format,
log_format=log_format)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
setup_loghandlers_from_args(args)
setup_redis(args)
cleanup_ghosts()
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.pid:
with open(os.path.expanduser(args.pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(args)
setup_redis(args)
cleanup_ghosts()
worker_class = import_attribute(args.worker_class)
try:
queues = list(map(Queue, args.queues))
w = worker_class(queues,
name=args.name,
default_worker_ttl=args.worker_ttl,
default_result_ttl=args.results_ttl)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
