def start_worker(self, node=HOSTNAME, logging_level="INFO"): if manager.debug: logger.warn("running in debug mode") return name = "{0}:{1}".format(node, uuid.uuid4().hex[:8]) q_names = self.queues cleanup_ghosts(manager.connection) rqs = [ Queue(queue, connection=manager.connection) for queue in q_names ] worker = FlaskWorker( rqs, name=name, connection=manager.connection, default_worker_ttl=manager.worker_ttl, default_result_ttl=manager.result_ttl, job_class=Job, queue_class=Queue, exception_handlers=None, ) if manager.sentry_dsn: client = Client(manager.sentry_dsn) register_sentry(client, worker) worker.work(burst=False, logging_level=logging_level)
def auto_worker(queue_name, redis_url, skip_failed, default_result_ttl): """ Internal target to use in multiprocessing """ connection = Redis.from_url(redis_url) cleanup_ghosts(connection) worker_class = import_attribute("rq.Worker") queue_class = import_attribute("rq.Queue") queue = queue_class(queue_name, connection=connection) if skip_failed: exception_handlers = [] else: exception_handlers = None name = "{}-auto".format(uuid4().hex) worker = worker_class( queue, name=name, connection=connection, exception_handlers=exception_handlers, default_result_ttl=default_result_ttl, ) worker.work(burst=True)
def worker(cli_config, burst, logging_level, name, results_ttl, worker_ttl, job_monitoring_interval, verbose, quiet, sentry_dsn, exception_handler, pid, queues, rate_limit, **options): """Starts an RQ worker.""" settings = read_config_file(cli_config.config) if cli_config.config else {} # Worker specific default arguments queues = queues or settings.get('QUEUES', ['default']) sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN') if pid: with open(os.path.expanduser(pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(verbose, quiet) try: cleanup_ghosts(cli_config.connection) exception_handlers = [] for h in exception_handler: exception_handlers.append(import_attribute(h)) if is_suspended(cli_config.connection): click.secho( 'RQ is currently suspended, to resume job execution run "rq resume"', fg='red') sys.exit(1) queues = [ cli_config.queue_class(queue, connection=cli_config.connection, job_class=cli_config.job_class) for queue in queues ] worker = cli_config.worker_class( queues, name=name, connection=cli_config.connection, default_worker_ttl=worker_ttl, default_result_ttl=results_ttl, job_monitoring_interval=job_monitoring_interval, job_class=cli_config.job_class, queue_class=cli_config.queue_class, exception_handlers=exception_handlers or None, limit=rate_limit) # Should we configure Sentry? if sentry_dsn: from raven import Client from raven.transport.http import HTTPTransport from rq.contrib.sentry import register_sentry client = Client(sentry_dsn, transport=HTTPTransport) register_sentry(client, worker) worker.work(burst=burst, logging_level=logging_level) except ConnectionError as e: print(e) sys.exit(1)
def worker(url, config, burst, name, worker_class, job_class, queue_class, connection_class, path, results_ttl, worker_ttl, verbose, quiet, sentry_dsn, exception_handler, pid, queues): """Starts an RQ worker.""" if path: sys.path = path.split(':') + sys.path settings = read_config_file(config) if config else {} # Worker specific default arguments queues = queues or settings.get('QUEUES', ['default']) sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN') if pid: with open(os.path.expanduser(pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(verbose, quiet) connection_class = import_attribute(connection_class) conn = connect(url, config, connection_class) cleanup_ghosts(conn) worker_class = import_attribute(worker_class) queue_class = import_attribute(queue_class) exception_handlers = [] for h in exception_handler: exception_handlers.append(import_attribute(h)) if is_suspended(conn): click.secho( 'RQ is currently suspended, to resume job execution run "rq resume"', fg='red') sys.exit(1) try: queues = [queue_class(queue, connection=conn) for queue in queues] w = worker_class(queues, name=name, connection=conn, default_worker_ttl=worker_ttl, default_result_ttl=results_ttl, job_class=job_class, queue_class=queue_class, exception_handlers=exception_handlers or None) # Should we configure Sentry? if sentry_dsn: from raven import Client from raven.transport.http import HTTPTransport from rq.contrib.sentry import register_sentry client = Client(sentry_dsn, transport=HTTPTransport) register_sentry(client, w) w.work(burst=burst) except ConnectionError as e: print(e) sys.exit(1)
def worker(cli_config, burst, logging_level, name, results_ttl, worker_ttl, job_monitoring_interval, disable_job_desc_logging, verbose, quiet, sentry_dsn, exception_handler, pid, disable_default_exception_handler, max_jobs, with_scheduler, queues, log_format, date_format, **options): """Starts an RQ worker.""" settings = read_config_file(cli_config.config) if cli_config.config else {} # Worker specific default arguments queues = queues or settings.get('QUEUES', ['default']) sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN') name = name or settings.get('NAME') if pid: with open(os.path.expanduser(pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(verbose, quiet, date_format, log_format) try: cleanup_ghosts(cli_config.connection) exception_handlers = [] for h in exception_handler: exception_handlers.append(import_attribute(h)) if is_suspended(cli_config.connection): click.secho('RQ is currently suspended, to resume job execution run "rq resume"', fg='red') sys.exit(1) queues = [cli_config.queue_class(queue, connection=cli_config.connection, job_class=cli_config.job_class) for queue in queues] worker = cli_config.worker_class( queues, name=name, connection=cli_config.connection, default_worker_ttl=worker_ttl, default_result_ttl=results_ttl, job_monitoring_interval=job_monitoring_interval, job_class=cli_config.job_class, queue_class=cli_config.queue_class, exception_handlers=exception_handlers or None, disable_default_exception_handler=disable_default_exception_handler, log_job_description=not disable_job_desc_logging ) # Should we configure Sentry? if sentry_dsn: from rq.contrib.sentry import register_sentry register_sentry(sentry_dsn) # if --verbose or --quiet, override --logging_level if verbose or quiet: logging_level = None worker.work(burst=burst, logging_level=logging_level, date_format=date_format, log_format=log_format, max_jobs=max_jobs, with_scheduler=with_scheduler) except ConnectionError as e: print(e) sys.exit(1)
def worker(url, config, burst, name, worker_class, job_class, queue_class, path, results_ttl, worker_ttl, verbose, quiet, sentry_dsn, exception_handler, pid, queues): """Starts an RQ worker.""" if path: sys.path = path.split(':') + sys.path settings = read_config_file(config) if config else {} # Worker specific default arguments queues = queues or settings.get('QUEUES', ['default']) sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN') if pid: with open(os.path.expanduser(pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(verbose, quiet) conn = connect(url, config) cleanup_ghosts(conn) worker_class = import_attribute(worker_class) queue_class = import_attribute(queue_class) exception_handlers = [] for h in exception_handler: exception_handlers.append(import_attribute(h)) if is_suspended(conn): click.secho('RQ is currently suspended, to resume job execution run "rq resume"', fg='red') sys.exit(1) try: queues = [queue_class(queue, connection=conn) for queue in queues] w = worker_class(queues, name=name, connection=conn, default_worker_ttl=worker_ttl, default_result_ttl=results_ttl, job_class=job_class, queue_class=queue_class, exception_handlers=exception_handlers or None) # Should we configure Sentry? if sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry from raven.transport.http import HTTPTransport client = Client(sentry_dsn, transport=HTTPTransport) register_sentry(client, w) w.work(burst=burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.pid: with open(os.path.expanduser(args.pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(args) setup_redis(args) cleanup_ghosts() worker_class = import_attribute(args.worker_class) queue_class = import_attribute(args.queue_class) try: queues = list(map(queue_class, args.queues)) w = worker_class(queues, name=args.name, default_worker_ttl=args.worker_ttl, default_result_ttl=args.results_ttl, job_class=args.job_class) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def worker(url, config, burst, name, worker_class, job_class, queue_class, path, results_ttl, worker_ttl, verbose, quiet, sentry_dsn, pid, queues): """Starts an RQ worker.""" if path: sys.path = path.split(':') + sys.path settings = read_config_file(config) if config else {} # Worker specific default arguments url = url or settings.get('REDIS_URL') queues = queues or settings.get('QUEUES', ['default']) sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN') if pid: with open(os.path.expanduser(pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(verbose, quiet) conn = connect(url) cleanup_ghosts(conn) worker_class = import_attribute(worker_class) queue_class = import_attribute(queue_class) if worker_class.paused(): click.secho("The worker has been paused, run reset_paused", fg='red') sys.exit(1) try: queues = [queue_class(queue, connection=conn) for queue in queues] w = worker_class(queues, name=name, connection=conn, default_worker_ttl=worker_ttl, default_result_ttl=results_ttl, job_class=job_class) # Should we configure Sentry? if sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(sentry_dsn) register_sentry(client, w) w.work(burst=burst) except ConnectionError as e: print(e) sys.exit(1)
def worker(cli_config, burst, logging_level, name, results_ttl, worker_ttl, job_monitoring_interval, verbose, quiet, sentry_dsn, exception_handler, pid, disable_default_exception_handler, queues, log_format, date_format, **options): """Starts an RQ worker.""" settings = read_config_file(cli_config.config) if cli_config.config else {} # Worker specific default arguments queues = queues or settings.get('QUEUES', ['default']) sentry_dsn = sentry_dsn or settings.get('SENTRY_DSN') name = name or settings.get('NAME') if pid: with open(os.path.expanduser(pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(verbose, quiet, date_format, log_format) try: cleanup_ghosts(cli_config.connection) exception_handlers = [] for h in exception_handler: exception_handlers.append(import_attribute(h)) if is_suspended(cli_config.connection): click.secho('RQ is currently suspended, to resume job execution run "rq resume"', fg='red') sys.exit(1) queues = [cli_config.queue_class(queue, connection=cli_config.connection, job_class=cli_config.job_class) for queue in queues] worker = cli_config.worker_class( queues, name=name, connection=cli_config.connection, default_worker_ttl=worker_ttl, default_result_ttl=results_ttl, job_monitoring_interval=job_monitoring_interval, job_class=cli_config.job_class, queue_class=cli_config.queue_class, exception_handlers=exception_handlers or None, disable_default_exception_handler=disable_default_exception_handler ) # Should we configure Sentry? if sentry_dsn: from rq.contrib.sentry import register_sentry register_sentry(sentry_dsn) worker.work(burst=burst, logging_level=logging_level, date_format=date_format, log_format=log_format) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.pid: with open(os.path.expanduser(args.pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(args) setup_redis(args) cleanup_ghosts() worker_class = import_attribute(args.worker_class) try: queues = list(map(Queue, args.queues)) w = worker_class(queues, name=args.name, default_worker_ttl=args.worker_ttl, default_result_ttl=args.results_ttl, job_class=args.job_class) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Other default arguments if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.verbose and args.quiet: raise RuntimeError( "Flags --verbose and --quiet are mutually exclusive.") if args.verbose: level = 'DEBUG' elif args.quiet: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) setup_redis(args) cleanup_ghosts() try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.pid: with open(os.path.expanduser(args.pid), "w") as fp: fp.write(str(os.getpid())) # 把 pid 写到 home 目录下 setup_loghandlers_from_args(args) setup_redis(args) # 启动 redis,创建好 connection 压入栈內,下面的 Queue、Worker 后续都会取到 cleanup_ghosts() # 清理 ghost worker worker_class = import_attribute(args.worker_class) try: queues = list(map(Queue, args.queues)) # 实例化 queue,默认是 default queue w = worker_class(queues, name=args.name) # 实例化 worker # Should we configure Sentry? # raven 是 Sentry(用于监控代码发生异常时通知维护人员)的 python 客户端 if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) # 启动 worker except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Other default arguments if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.verbose and args.quiet: raise RuntimeError("Flags --verbose and --quiet are mutually exclusive.") if args.verbose: level = 'DEBUG' elif args.quiet: level = 'WARNING' else: level = 'INFO' setup_loghandlers(level) setup_redis(args) cleanup_ghosts() try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', None) setup_loghandlers_from_args(args) setup_redis(args) cleanup_ghosts() try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
def run(name: str, queue: str, connection_host: str, connection_port: int = 6379, connection_password: str = None, connection_over_tls: bool = False, connection_db: int = 0, verbose: bool = False): date_format = "%Y-%m-%dT%H:%M:%S" log_format = "%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s" setup_loghandlers_from_args(verbose, False, date_format, log_format) params = { "ssl": connection_over_tls, "password": connection_password, "host": connection_host, "port": connection_port, "db": connection_db } try: c = Redis(**params) cleanup_ghosts(c) worker = Worker([queue], name=name, connection=c, default_worker_ttl=420, default_result_ttl=500, job_monitoring_interval=30, job_class=Job, queue_class=Queue, exception_handlers=None) worker.work(burst=False, logging_level="DEBUG" if verbose else "INFO", date_format=date_format, log_format=log_format) except ConnectionError as e: print(e) sys.exit(1)
def main(): args = parse_args() if args.path: sys.path = args.path.split(':') + sys.path settings = {} if args.config: settings = read_config_file(args.config) setup_default_arguments(args, settings) # Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) setup_loghandlers_from_args(args) setup_redis(args) cleanup_ghosts() try: queues = map(Queue, args.queues) w = Worker(queues, name=args.name) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w) w.work(burst=args.burst) except ConnectionError as e: print(e) sys.exit(1)
# Worker specific default arguments if not args.queues: args.queues = settings.get('QUEUES', ['default']) if args.sentry_dsn is None: args.sentry_dsn = settings.get('SENTRY_DSN', os.environ.get('SENTRY_DSN', None)) if args.pid: with open(os.path.expanduser(args.pid), "w") as fp: fp.write(str(os.getpid())) setup_loghandlers_from_args(args) setup_redis(args) cleanup_ghosts() worker_class = import_attribute(args.worker_class) try: queues = list(map(Queue, args.queues)) w = worker_class(queues, name=args.name, default_worker_ttl=args.worker_ttl, default_result_ttl=args.results_ttl) # Should we configure Sentry? if args.sentry_dsn: from raven import Client from rq.contrib.sentry import register_sentry client = Client(args.sentry_dsn) register_sentry(client, w)