def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) setup_redis(args) try: if args.only_queues: func = show_queues elif args.only_workers: func = show_workers else: func = show_both interval(args.interval, func, args) except ConnectionError as e: print(e) sys.exit(1) except KeyboardInterrupt: print sys.exit(0)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Other default arguments if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', None) setup_loghandlers(args) setup_redis(args) try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Other default arguments if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', None) setup_loghandlers(args) setup_redis(args) try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) w.push_exc_handler(do_job_failure_handler_have_a_rest) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) setup_redis(args) try: if args.empty_failed_queue: num_jobs = get_failed_queue().empty() print('{} jobs removed from failed queue'.format(num_jobs)) else: if args.only_queues: func = show_queues elif args.only_workers: func = show_workers else: func = show_both interval(args.interval, func, args) except ConnectionError as e: print(e) sys.exit(1) except KeyboardInterrupt: print() sys.exit(0)
def setup_redis_with_arguments(self, arguments): parser = rqworker.setup_parser() args = parser.parse_args(arguments) setup_default_arguments(args, {}) self.current_connection = connections.get_current_connection() setup_redis(args)
def test_rqinfo_can_choose_queue(self): parser = rqinfo.setup_parser() args = parser.parse_args(self.base_arguments + ["B"]) setup_default_arguments(args, {}) setup_redis(args) output = self.capture_stdout(rqinfo.show_queues, args) connections.use_connection(self.testconn) expected_output = ["queue B 1"] self.assertEqual(output.splitlines(), expected_output)
def test_rqinfo_defaults_to_all_queues(self): parser = rqinfo.setup_parser() args = parser.parse_args(self.base_arguments) setup_default_arguments(args, {}) setup_redis(args) # This uses use_connection and clears self.testconn from the stack output = self.capture_stdout(rqinfo.show_queues, args) connections.use_connection(self.testconn) expected_output = set(["queue B 1", "queue C 1", "queue A 1"]) self.assertEqual(set(output.splitlines()), expected_output)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.pid: with open(os.path.expanduser(args.pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(args) setup_redis(args) cleanup_ghosts() worker_class = import_attribute(args.worker_class) queue_class = import_attribute(args.queue_class) try: queues = list(map(queue_class, args.queues)) w = worker_class(queues, name=args.name, default_worker_ttl=args.worker_ttl, default_result_ttl=args.results_ttl, job_class=args.job_class) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.pid: with open(os.path.expanduser(args.pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(args) setup_redis(args) cleanup_ghosts() worker_class = import_attribute(args.worker_class) try: queues = list(map(Queue, args.queues)) w = worker_class(queues, name=args.name, default_worker_ttl=args.worker_ttl, default_result_ttl=args.results_ttl, job_class=args.job_class) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Other default arguments if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.verbose and args.quiet: raise RuntimeError( "Flags --verbose and --quiet are mutually exclusive.") if args.verbose: level = 'DEBUG' elif args.quiet: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) setup_redis(args) cleanup_ghosts() try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Other default arguments if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.verbose and args.quiet: raise RuntimeError("Flags --verbose and --quiet are mutually exclusive.") if args.verbose: level = 'DEBUG' elif args.quiet: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) setup_redis(args) cleanup_ghosts() try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.pid: with open(os.path.expanduser(args.pid), "w") as fp: fp.write(str(os.getpid())) # 把 pid 写到 home 目录下 setup_loghandlers_from_args(args) setup_redis(args) # 启动 redis,创建好 connection 压入栈內,下面的 Queue、Worker 后续都会取到 cleanup_ghosts() # 清理 ghost worker worker_class = import_attribute(args.worker_class) try: queues = list(map(Queue, args.queues)) # 实例化 queue,默认是 default queue w = worker_class(queues, name=args.name) # 实例化 worker # Should we configure Sentry? # raven 是 Sentry(用于监控代码发生异常时通知维护人员)的 python 客户端 if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) # 启动 worker except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) setup_loghandlers_from_args(args) setup_redis(args) cleanup_ghosts() try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
# Can't find queue, which should basically never happen as we only work jobs that match the given queue names and # queues are transient in rq. logger.warn('job %s: cannot find queue %s - moving to failed queue' % (job.id, job.origin)) return True args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.pid: with open(os.path.expanduser(args.pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(args) setup_redis(args)
#!/usr/bin/env python from flask import _request_ctx_stack from rq import Worker, Queue from rq.scripts.rqworker import setup_redis, parse_args from rq.scripts import setup_default_arguments from gitorama import app from gitorama.flask_logbook import create_logger class CustomWorker(Worker): def perform_job(self, job): with app.test_request_context(): _request_ctx_stack.top.logbook_request_id = job.get_id() handlers = create_logger(app) with handlers: super(CustomWorker, self).perform_job(job) args = parse_args() setup_default_arguments(args, {}) setup_redis(args) queue = Queue() w = CustomWorker([queue]) w.work(burst=args.burst)