def main(): max_workers = 5 if "GOL_WORKERS" in os.environ: max_workers = int(os.getenv("GOL_WORKERS")) processes = [] q = None for _ in range(0, 3): try: rq.use_connection(redis.Redis()) q = rq.Queue("lifeboxQueue") break except Exception: time.sleep(5) if q is None: print("Could not establish connection to redis, exiting") sys.exit(99) for _ in range(0, max_workers): try: proc = multiprocessing.Process(target=rq.Worker(q).work) proc.start() processes.append(proc) except Exception as ex: print(ex) traceback.print_exc() try: while True: tbd = [] for proc in processes: if not proc.is_alive(): try: tbd.append(proc) newproc = multiprocessing.Process( target=rq.Worker(q).work) newproc.start() processes.append(newproc) except Exception as ex: print(ex) traceback.print_exc() for proc in tbd: processes.remove(proc) time.sleep(10) except KeyboardInterrupt: print("interrupred, exiting and killing") pass finally: for proc in processes: proc.kill()
def run(self): """ Run the queue-worker """ with rq.Connection(): # start the task worker self.worker = rq.Worker(self.q) self.worker.work() print('Worker was startetd')
def main(): # check deps planner_url = os.environ.get('KRAKEN_PLANNER_URL', consts.DEFAULT_PLANNER_URL) srvcheck.check_url('planner', planner_url, 7997) redis_addr = os.environ.get('KRAKEN_REDIS_ADDR', consts.DEFAULT_REDIS_ADDR) srvcheck.check_tcp_service('redis', redis_addr, 6379) rds = redis.Redis(host=redis_addr, port=6379, db=consts.REDIS_RQ_DB) db_url = os.environ.get('KRAKEN_DB_URL', consts.DEFAULT_DB_URL) srvcheck.check_postgresql(db_url) logs.setup_logging('rq') log.info('Kraken RQ started, version %s', version.version) # Create Flask app instance app = Flask('Kraken RQ') app.config["SQLALCHEMY_ECHO"] = False app.config["SQLALCHEMY_DATABASE_URI"] = db_url + '?application_name=rq' app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False # initialize SqlAlchemy db.init_app(app) # setup sentry with app.app_context(): sentry_url = get_setting('monitoring', 'sentry_dsn') logs.setup_sentry(sentry_url) worker = rq.Worker('kq', connection=rds, exception_handlers=[_exception_handler]) worker.work()
def test_transport_shutdown(sentry_init): sentry_init(integrations=[RqIntegration()]) events_r, events_w = os.pipe() events_r = os.fdopen(events_r, "rb", 0) events_w = os.fdopen(events_w, "wb", 0) def capture_event(event): events_w.write(json.dumps(event).encode("utf-8")) events_w.write(b"\n") def flush(timeout=None, callback=None): events_w.write(b"flush\n") Hub.current.client.transport.capture_event = capture_event Hub.current.client.flush = flush queue = rq.Queue(connection=FakeStrictRedis()) worker = rq.Worker([queue], connection=queue.connection) queue.enqueue(crashing_job, foo=42) worker.work(burst=True) event = events_r.readline() event = json.loads(event.decode("utf-8")) exception, = event["exception"]["values"] assert exception["type"] == "ZeroDivisionError" assert events_r.readline() == b"flush\n"
def run(): q = None queue_name = raw_input() try: redis_url = settings.STRUCTURE_REDIS except: redis_url = None connection = redis.Redis.from_url(redis_url) too_high_queue = rq.Queue(Consts.QUEUE_PRIORITY_TOO_HIGH, connection=connection) high_queue = rq.Queue(Consts.QUEUE_PRIORITY_HIGH, connection=connection) normal_queue = rq.Queue(Consts.QUEUE_PRIORITY_NORMAL, connection=connection) low_queue = rq.Queue(Consts.QUEUE_PRIORITY_LOW, connection=connection) if queue_name == "structure:higher": q = too_high_queue elif queue_name == "structure:high": q = high_queue elif queue_name == "structure:normal": q = normal_queue elif queue_name == "structure:low": q = low_queue else: print "Error: Cannot find the queue from Redis." if q is not None: with rq.Connection(connection): w = rq.Worker([q]) w.work()
def run(self, redis): redis_connection = Redis(redis[0], redis[1], password=redis[2]) with Connection(redis_connection): qs = map(Queue, self.queues) or [Queue()] w = rq.Worker(qs) w.work() print('Items in queue \'{0}\': {1}'.format(self.queues[0], len(qs)))
def main(): utils.setup_logging() config.log() gh_pr.monkeypatch_github() if config.FLUSH_REDIS_ON_STARTUP: utils.get_redis().flushall() with rq.Connection(utils.get_redis()): worker = rq.Worker(['default']) worker.work()
def work(): """Start an rq worker on the connection provided by create_connection.""" # Preload ROOT module to reduce worker startup time import ROOT # noqa with rq.Connection(create_connection()): worker = rq.Worker(list(map(create_queue, QUEUES))) # Quiet workers to suppress large result output # https://github.com/nvie/rq/issues/136 worker.log.setLevel(logging.WARNING) worker.work()
def main(): """Sets up Redis connection and starts the worker.""" redis_connection = redis.Redis(host="queue-server") with rq.Connection(redis_connection): queue = rq.Queue('build_n_run_queue') worker = rq.Worker([queue]) while queue.count + queue.deferred_job_registry.count > 0: worker.work(burst=True) time.sleep(5)
def start_worker( queue_name, redis_url, with_scheduler=False ): redis_conn = redis.from_url( redis_url ) with rq.Connection( redis_conn ): worker = rq.Worker( queues = [queue_name], connection = redis_conn ) worker.work( with_scheduler=with_scheduler )
def main(): # pragma: no cover utils.setup_logging() config.log() gh_pr.monkeypatch_github() r = utils.get_redis_for_rq() if config.FLUSH_REDIS_ON_STARTUP: r.flushall() with rq.Connection(r): worker = rq.Worker(['default']) if config.SENTRY_URL: client = raven.Client(config.SENTRY_URL, transport=HTTPTransport) register_sentry(client, worker) worker.work()
def command(self): self._load_config() queue.rq_config.in_worker = True connection = queue.rq_config.connection if not connection: log.error('No redis connection available') exit(1) queue_ = queue.rq_config.queue if not queue_: log.error('No queue available.') exit(1) worker = rq.Worker([queue_], connection=connection) worker.work()
def load_app(config): database = config.get('database') queues = config.get('queues', ['default']) redis_conn = redis.from_url(database) hostname = socket.gethostname() name = '%s-%s' % (hostname, uuid.uuid4().hex) with rq.Connection(redis_conn): worker = rq.Worker(map(rq.Queue, queues), name=name) return worker
def main(argv): # CONNECT TO DATABASE # UNCOMMENT NEXT 4 LINES # connect("extensionsASTnpantel", username="******", host="localhost", port=37017) # load data from query # queue = readFromDatabase() # queuetest = ['a','b','c','d','e','f','g','h','i','j','k','l'] # put them and run them in the queue # specialLCSHandler(queue) with rq.Connection(redis.Redis(REDIS_HOST, REDIS_PORT)) as connection: # print(job.result) w = rq.Worker([QUEUE_NAME]) w.work()
def test_transport_shutdown(sentry_init, capture_events_forksafe): sentry_init(integrations=[RqIntegration()]) events = capture_events_forksafe() queue = rq.Queue(connection=FakeStrictRedis()) worker = rq.Worker([queue], connection=queue.connection) queue.enqueue(crashing_job, foo=42) worker.work(burst=True) event = events.read_event() events.read_flush() (exception, ) = event["exception"]["values"] assert exception["type"] == "ZeroDivisionError"
def run(self, args): redis = StrictRedis(config.REDIS_HOST, config.REDIS_PORT) workers = [] queue_config = parse_queue_config(args.config) with rq.Connection(redis): for name, count in queue_config: queue = rq.Queue(name) for i in range(count): w = rq.Worker([queue], name='{}:{}'.format(name, i)) workers.append(w) procs = [Process(target=start_worker, args=(w, )) for w in workers] for p in procs: p.start() for p in procs: p.join()
#!/usr/bin/env python import sys import rq # Preload libraries from app import router router.resolve_node_modules() # Provide queue names to listen to as arguments to this script, # similar to rqworker with rq.Connection(): qs = map(rq.Queue, sys.argv[1:]) or [rq.Queue()] w = rq.Worker(qs) w.work()
import rq # type: ignore from redis import Redis import os import init_log import logging if __name__ == "__main__": init_log.config_worker_log() # Tell rq what Redis connection to use with rq.Connection( Redis.from_url(os.environ.get("REDIS_URL") or "redis://")): q = rq.Queue() rq.Worker(q).work()
parser.add_argument( "--burst", action="store_const", const=True, default=False, help="enable burst mode") args = parser.parse_args() wiring = backend.wiring.Wiring() class JobWithWiring(Job): @property def kwargs(self): result = dict(super().kwargs) result["wiring"] = backend.wiring.Wiring() return result @kwargs.setter def kwargs(self, value): super().kwargs = value with rq.Connection(wiring.redis): w = rq.Worker( queues=[wiring.settings.TASK_QUEUE_NAME], name=uuid.uuid4().hex, job_class=JobWithWiring) w.work(burst=args.burst)
def run_workers_py(q): for i in xrange(2): print 'Started worker {}'.format(i) worker = rq.Worker([q], connection=q.connection) worker.work(burst=True) # Runs enqueued job
def exec_task(self, queue=ExtracterConsts.QUEUE_PRIORITY_NORMAL): with rq.Connection(): w = rq.Worker([queue]) w.work()
# Too many failures if job.meta['failures'] >= MAX_FAILURES: logger.warn( 'job %s: failed too many times times - moving to failed queue' % job.id) job.save() return True # Requeue job and stop it from being moved into the failed queue logger.warn('job %s: failed %d times - retrying' % (job.id, job.meta['failures'])) for queue in queues: if queue.name == job.origin: queue.enqueue_job(job, timeout=job.timeout) return False # Can't find queue, which should basically never happen as we only work jobs that match the given queue names and # queues are transient in rq. logger.warn('job %s: cannot find queue %s - moving to failed queue' % (job.id, job.origin)) return True with rq.Connection(): queues = map(rq.Queue, sys.argv[1:]) or [rq.Queue()] worker = rq.Worker(queues) worker.push_exc_handler(retry_handler) worker.work()
# Workaround current bug in docutils: # http://permalink.gmane.org/gmane.text.docutils.devel/6324 import docutils.utils # Make sure our PyPI directory is on the sys.path root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path = [root] + sys.path import config conf = config.Config( os.environ.get("PYPI_CONFIG", os.path.join(root, "config.ini"))) redis_conn = redis.Redis.from_url(conf.queue_redis_url) # Create our queues if sys.argv[1:]: queues = [rq.Queue(name, connection=redis_conn) for name in sys.argv[1:]] else: queues = [rq.Queue(connection=redis_conn)] # Create our Worker worker = rq.Worker(queues, connection=redis_conn) # Create our Sentry Client if conf.sentry_dsn: raven_client = raven.Client(conf.sentry_dsn) rq.contrib.sentry.register_sentry(raven_client, worker) # Run our worker, fetching jobs from the queue worker.work()
def main(): with rq.Connection(redis.from_url(REDIS_URL)): worker = rq.Worker(['default']) worker.work()
import os import redis import rq RQ_REDIS_URL = "redis://redis_server" with rq.Connection(redis.from_url(RQ_REDIS_URL)): worker = rq.Worker(['default']) worker.work()
action="store_const", const=True, default=False, help="enable burst mode") args = parser.parse_args() # Нам нужны настройки и подключение к Redis. wiring = backend.wiring.Wiring() class JobWithWiring(Job): @property def kwargs(self): result = dict(super().kwargs) result["wiring"] = backend.wiring.Wiring() return result @kwargs.setter def kwargs(self, value): super().kwargs = value with rq.Connection(wiring.redis): w = rq.Worker( queues=[wiring.settings.TASK_QUEUE_NAME], # Если мы захотим запускать несколько воркеров в разных # контейнерах, им потребуются уникальные имена. name=uuid.uuid4().hex, job_class=JobWithWiring) w.work(burst=args.burst)
def run_worker(): with rq.Connection(): qs = 'labmanager_unittests' w = rq.Worker(qs) w.work()
def run_worker(): print("WORKING") worker = rq.Worker([queue], connection=queue.connection) worker.work()
import os import redis import rq listen = ['high', 'default', 'low'] redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379') connection = redis.from_url(redis_url) if __name__ == '__main__': with rq.Connection(connection): worker = rq.Worker(map(rq.Queue, listen)) worker.work()
def worker(self): """Return the worker class.""" if not self._worker: self._worker = rq.Worker(queues=self._queues, name=self._name) return self._worker