def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
setup_redis(args)
try:
if args.only_queues:
func = show_queues
elif args.only_workers:
func = show_workers
else:
func = show_both
interval(args.interval, func, args)
except ConnectionError as e:
print(e)
sys.exit(1)
except KeyboardInterrupt:
print
sys.exit(0)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Other default arguments
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN', None)
setup_loghandlers(args)
setup_redis(args)
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Other default arguments
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN', None)
setup_loghandlers(args)
setup_redis(args)
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
w.push_exc_handler(do_job_failure_handler_have_a_rest)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
setup_redis(args)
try:
if args.only_queues:
func = show_queues
elif args.only_workers:
func = show_workers
else:
func = show_both
interval(args.interval, func, args)
except ConnectionError as e:
print(e)
sys.exit(1)
except KeyboardInterrupt:
print
sys.exit(0)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
setup_redis(args)
try:
if args.empty_failed_queue:
num_jobs = get_failed_queue().empty()
print('{} jobs removed from failed queue'.format(num_jobs))
else:
if args.only_queues:
func = show_queues
elif args.only_workers:
func = show_workers
else:
func = show_both
interval(args.interval, func, args)
except ConnectionError as e:
print(e)
sys.exit(1)
except KeyboardInterrupt:
print()
sys.exit(0)
def setup_redis_with_arguments(self, arguments):
parser = rqworker.setup_parser()
args = parser.parse_args(arguments)
setup_default_arguments(args, {})
self.current_connection = connections.get_current_connection()
setup_redis(args)
def test_rqinfo_can_choose_queue(self):
parser = rqinfo.setup_parser()
args = parser.parse_args(self.base_arguments + ["B"])
setup_default_arguments(args, {})
setup_redis(args)
output = self.capture_stdout(rqinfo.show_queues, args)
connections.use_connection(self.testconn)
expected_output = ["queue B 1"]
self.assertEqual(output.splitlines(), expected_output)
def test_rqinfo_defaults_to_all_queues(self):
parser = rqinfo.setup_parser()
args = parser.parse_args(self.base_arguments)
setup_default_arguments(args, {})
setup_redis(args) # This uses use_connection and clears self.testconn from the stack
output = self.capture_stdout(rqinfo.show_queues, args)
connections.use_connection(self.testconn)
expected_output = set(["queue B 1", "queue C 1", "queue A 1"])
self.assertEqual(set(output.splitlines()), expected_output)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.pid:
with open(os.path.expanduser(args.pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(args)
setup_redis(args)
cleanup_ghosts()
worker_class = import_attribute(args.worker_class)
queue_class = import_attribute(args.queue_class)
try:
queues = list(map(queue_class, args.queues))
w = worker_class(queues,
name=args.name,
default_worker_ttl=args.worker_ttl,
default_result_ttl=args.results_ttl,
job_class=args.job_class)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.pid:
with open(os.path.expanduser(args.pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(args)
setup_redis(args)
cleanup_ghosts()
worker_class = import_attribute(args.worker_class)
try:
queues = list(map(Queue, args.queues))
w = worker_class(queues,
name=args.name,
default_worker_ttl=args.worker_ttl,
default_result_ttl=args.results_ttl,
job_class=args.job_class)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Other default arguments
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.verbose and args.quiet:
raise RuntimeError(
"Flags --verbose and --quiet are mutually exclusive.")
if args.verbose:
level = 'DEBUG'
elif args.quiet:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
setup_redis(args)
cleanup_ghosts()
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Other default arguments
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.verbose and args.quiet:
raise RuntimeError("Flags --verbose and --quiet are mutually exclusive.")
if args.verbose:
level = 'DEBUG'
elif args.quiet:
level = 'WARNING'
else:
level = 'INFO'
setup_loghandlers(level)
setup_redis(args)
cleanup_ghosts()
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.pid:
with open(os.path.expanduser(args.pid), "w") as fp:
fp.write(str(os.getpid())) # 把 pid 写到 home 目录下
setup_loghandlers_from_args(args)
setup_redis(args) # 启动 redis，创建好 connection 压入栈內，下面的 Queue、Worker 后续都会取到
cleanup_ghosts() # 清理 ghost worker
worker_class = import_attribute(args.worker_class)
try:
queues = list(map(Queue, args.queues)) # 实例化 queue，默认是 default queue
w = worker_class(queues, name=args.name) # 实例化 worker
# Should we configure Sentry?
# raven 是 Sentry（用于监控代码发生异常时通知维护人员）的 python 客户端
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst) # 启动 worker
except ConnectionError as e:
print(e)
sys.exit(1)
def main():
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
setup_loghandlers_from_args(args)
setup_redis(args)
cleanup_ghosts()
try:
queues = map(Queue, args.queues)
w = Worker(queues, name=args.name)
# Should we configure Sentry?
if args.sentry_dsn:
from raven import Client
from rq.contrib.sentry import register_sentry
client = Client(args.sentry_dsn)
register_sentry(client, w)
w.work(burst=args.burst)
except ConnectionError as e:
print(e)
sys.exit(1)
# Can't find queue, which should basically never happen as we only work jobs that match the given queue names and
# queues are transient in rq.
logger.warn('job %s: cannot find queue %s - moving to failed queue' % (job.id, job.origin))
return True
args = parse_args()
if args.path:
sys.path = args.path.split(':') + sys.path
settings = {}
if args.config:
settings = read_config_file(args.config)
setup_default_arguments(args, settings)
# Worker specific default arguments
if not args.queues:
args.queues = settings.get('QUEUES', ['default'])
if args.sentry_dsn is None:
args.sentry_dsn = settings.get('SENTRY_DSN',
os.environ.get('SENTRY_DSN', None))
if args.pid:
with open(os.path.expanduser(args.pid), "w") as fp:
fp.write(str(os.getpid()))
setup_loghandlers_from_args(args)
setup_redis(args)
#!/usr/bin/env python
from flask import _request_ctx_stack
from rq import Worker, Queue
from rq.scripts.rqworker import setup_redis, parse_args
from rq.scripts import setup_default_arguments
from gitorama import app
from gitorama.flask_logbook import create_logger
class CustomWorker(Worker):
def perform_job(self, job):
with app.test_request_context():
_request_ctx_stack.top.logbook_request_id = job.get_id()
handlers = create_logger(app)
with handlers:
super(CustomWorker, self).perform_job(job)
args = parse_args()
setup_default_arguments(args, {})
setup_redis(args)
queue = Queue()
w = CustomWorker([queue])
w.work(burst=args.burst)
#!/usr/bin/env python
from flask import _request_ctx_stack
from rq import Worker, Queue
from rq.scripts.rqworker import setup_redis, parse_args
from rq.scripts import setup_default_arguments
from gitorama import app
from gitorama.flask_logbook import create_logger
class CustomWorker(Worker):
def perform_job(self, job):
with app.test_request_context():
_request_ctx_stack.top.logbook_request_id = job.get_id()
handlers = create_logger(app)
with handlers:
super(CustomWorker, self).perform_job(job)
args = parse_args()
setup_default_arguments(args, {})
setup_redis(args)
queue = Queue()
w = CustomWorker([queue])
w.work(burst=args.burst)
